Skip to content

Commit 5c2ebc2

Browse files
author
David Noble
committed
examples/searchcommands_app update + improved unit tetsts + bug fixes
Feature changes + SimulateCommand Includes seed option that enables testing in a predictable way Improved unit tests + Updated test_*_in_isoltion + Stubbed test_*_on_server Bug fixes + Error message that "Static configuration is unsupported" is semantically incorrect Fix is in search_command.py at or about lines 188-190 + SearchCommandParser.unquote does not implement Splunk quote rules Fix is at or about lines 310-335 in search_command_internals.py. Fix is not ready for production, but does unblock Shakeel + Validators do not format the data types they represent. Fix is in flight. See validators.py
1 parent 6055353 commit 5c2ebc2

File tree

5 files changed

+90
-76
lines changed

5 files changed

+90
-76
lines changed

‎examples/searchcommands_app/bin/simulate.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ class SimulateCommand(GeneratingCommand):
7979
**Description:** Average event count during sampling `interval`''',
8080
require=True, validate=validators.Integer(1))
8181

82+
seed = Option(
83+
doc='''**Syntax:** **seed=***<string>*
84+
**Description:** Value for initializing the random number generator ''',
85+
require=False)
86+
8287
def generate(self):
8388
""" Yields one random record at a time for the duration of `duration` """
8489
self.logger.debug('SimulateCommand: %s' % self) # log command line

‎splunklib/searchcommands/search_command.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
# Absolute imports
1818
from collections import OrderedDict
1919
from inspect import getmembers
20-
from logging import getLogger
2120
from os import path
2221
from sys import argv, stdin, stdout
2322

@@ -157,6 +156,7 @@ def process(self, args=argv, input_file=stdin, output_file=stdout):
157156
return
158157
self._configuration = ConfigurationSettings(self)
159158
if self.show_configuration:
159+
self.logger.debug(str(self.configuration))
160160
self.messages.append('info_message', str(self.configuration))
161161
writer = csv.DictWriter(
162162
output_file, self, self.configuration.keys(), mv_delimiter=',')
@@ -185,9 +185,9 @@ def process(self, args=argv, input_file=stdin, output_file=stdout):
185185
'Static configuration is unsupported. Please configure this '
186186
'command as follows in default/commands.conf:\n\n'
187187
'[%s]\n'
188-
'filename = %s\n' %
189-
'supports_getinfo = true'
190-
(type(self).name, path.basename(argv[0])))
188+
'filename = %s\n'
189+
'supports_getinfo = true' %
190+
(type(self).__name__, path.basename(argv[0])))
191191
self.messages.append('error_message', message)
192192
self.messages.write(output_file)
193193
self.logger.error(message)

‎splunklib/searchcommands/search_command_internals.py

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,9 +307,32 @@ def unquote(cls, string):
307307
single-quoted strings ("'") in addition to double-quoted ('"') strings.
308308
309309
"""
310-
if string[0] == '"' and string[-1] == '"':
311-
string = string[1:-1]
312-
return re.sub(cls._escaped_quote_re, '"', string)
310+
if len(string) == 0:
311+
return ''
312+
313+
if string[0] != '"':
314+
return string
315+
316+
if len(string) == 1:
317+
return string
318+
319+
if string[-1] != '"':
320+
raise ValueError("Poorly formed string literal: %s" % string)
321+
322+
def replace(match):
323+
value = match.group(0)
324+
if value == '\\\\':
325+
return '\\'
326+
if value == '\\"':
327+
return '"'
328+
if value == '""':
329+
return '"'
330+
if len(value) != 2:
331+
raise ValueError("Poorly formed string literal: %s" % string)
332+
return value # consistent with python handling
333+
334+
result = re.sub(cls._escaped_quote_re, replace, string[1:-1])
335+
return result
313336

314337
#region Class variables
315338

@@ -329,7 +352,7 @@ def unquote(cls, string):
329352
\s*$
330353
""", re.VERBOSE)
331354

332-
_escaped_quote_re = re.compile(r"""(""|\\")""")
355+
_escaped_quote_re = re.compile(r"""(\\\\|\\"|""|\\."|\\])""")
333356

334357
_name_re = re.compile(r"""[_a-zA-Z][[_a-zA-Z0-9]+""")
335358

‎splunklib/searchcommands/validators.py

Lines changed: 24 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,18 @@
1919

2020
class Validator(object):
2121
""" TODO: Documentation
22+
2223
"""
2324
def __call__(self, value):
2425
raise NotImplementedError()
2526

27+
def format(self, value):
28+
raise NotImplementedError()
29+
2630

2731
class Boolean(Validator):
2832
""" TODO: Documentation
33+
2934
"""
3035
truth_values = {
3136
'1': True, '0': False,
@@ -46,6 +51,7 @@ def __call__(self, value):
4651

4752
class Duration(Validator):
4853
""" TODO: Documentation
54+
4955
"""
5056
def __call__(self, value):
5157
if value is not None:
@@ -59,6 +65,7 @@ def __call__(self, value):
5965

6066
class Fieldname(Validator):
6167
""" TODO: Documentation
68+
6269
"""
6370
import re
6471
pattern = re.compile(r'''[_.a-zA-Z-][_.a-zA-Z0-9-]*$''')
@@ -88,6 +95,9 @@ def __call__(self, value):
8895
% (value, self.mode, self.buffering, e))
8996
return value
9097

98+
def format(self, value):
99+
return value.name
100+
91101

92102
class Integer(Validator):
93103
""" TODO: Documentation
@@ -131,55 +141,10 @@ def __call__(self, value):
131141
value = re.compile(value)
132142
except re.error as e:
133143
raise ValueError('%s: %s' % (str(e).capitalize(), value))
134-
return RegularExpression.Pattern(value)
135-
136-
class Pattern(object):
137-
138-
def __init__(self, regex):
139-
self.regex = regex
140-
141-
def __repr__(self):
142-
return 'RegularExpression.Pattern(%s)' % self.regex
143-
144-
def __str__(self):
145-
return self.regex.pattern
146-
147-
@property
148-
def flags(self):
149-
return self.regex.flags
150-
151-
@property
152-
def groups(self):
153-
return self.regex.groups
154-
155-
@property
156-
def groupindex(self):
157-
return self.regex.groupindex
158-
159-
@property
160-
def pattern(self):
161-
return self.regex.pattern
162-
163-
def findall(self, string, pos=0, end=-1):
164-
return self.regex.findall(string, pos, end)
165-
166-
def finditer(self, string, pos=0, end=-1):
167-
return self.regex.finditer(string, pos, end)
168-
169-
def match(self, string, pos=0, end=-1):
170-
return self.regex.match(string, pos, end)
171-
172-
def search(self, string, pos=0, end=-1):
173-
return self.regex.search(string, pos, end)
174-
175-
def split(self, string, max_split=0):
176-
return self.regex.split(string, max_split)
177-
178-
def sub(self, repl, string, count=0):
179-
return self.regex.sub(string, repl, string, count)
144+
return value
180145

181-
def subn(self, repl, string, count=0):
182-
return self.regex.subn(string, repl, string, count)
146+
def format(self, value):
147+
return value.pattern
183148

184149

185150
class Set(Validator):
@@ -194,3 +159,14 @@ def __call__(self, value):
194159
if value not in self.membership:
195160
raise ValueError('Unrecognized value: %s' % value)
196161
return value
162+
163+
164+
class String(Validator):
165+
""" TODO: Documentation
166+
167+
"""
168+
def __call__(self, value):
169+
return str(value)
170+
171+
def format(self, value):
172+
return str(value)

‎tests/test_searchcommands_app.py

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -20,84 +20,92 @@
2020
import unittest
2121

2222
from subprocess import Popen
23-
import shutil
23+
import json
2424
import os
25+
import shutil
2526
import testlib
2627

2728

2829
class TestSearchCommandsApp(testlib.SDKTestCase):
2930

3031
def setUp(self):
3132
super(TestSearchCommandsApp, self).setUp()
32-
for directory in 'error', 'output':
33+
for directory in 'log', 'output':
3334
path = TestSearchCommandsApp._data_file(directory)
3435
if os.path.exists(path):
3536
shutil.rmtree(path)
3637
os.mkdir(path)
3738
return
3839

39-
def test_generating_command(self):
40+
def test_generating_command_in_isolation(self):
41+
encoder = json.JSONEncoder(ensure_ascii=False)
4042
self._run(
4143
'simulate', [
4244
'csv=%s' % TestSearchCommandsApp._data_file("input/population.csv"),
4345
'duration=00:00:10',
4446
'interval=00:00:01',
45-
'rate=200'],
47+
'rate=200',
48+
'seed=%s' % encoder.encode(TestSearchCommandsApp._seed)],
4649
__GETINFO__=(
4750
'input/population.csv',
4851
'output/samples.csv',
49-
'error/test_generating_command.log'),
52+
'log/test_generating_command_in_isolation.log'),
5053
__EXECUTE__=(
5154
'input/population.csv',
5255
'output/samples.csv',
53-
'error/test_generating_command.log')
54-
)
56+
'log/test_generating_command_in_isolation.log'))
5557
return
5658

57-
def test_reporting_command(self):
59+
def test_generating_command_on_server(self):
60+
pass
61+
62+
def test_reporting_command_in_isolation(self):
5863
self._run(
5964
'sum', [
6065
'__map__', 'total=total', 'count'],
6166
__GETINFO__=(
6267
'input/counts.csv',
6368
'output/subtotals.csv',
64-
'error/test_reporting_command.log'),
69+
'log/test_reporting_command_in_isolation.log'),
6570
__EXECUTE__=(
6671
'input/counts.csv',
6772
'output/subtotals.csv',
68-
'error/test_reporting_command.log')
69-
)
73+
'log/test_reporting_command_in_isolation.log'))
7074
self._run(
7175
'sum', [
7276
'total=total', 'count'],
7377
__GETINFO__=(
7478
'input/subtotals.csv',
7579
'output/totals.csv',
76-
'error/test_reporting_command.log'),
80+
'log/test_reporting_command_in_isolation.log'),
7781
__EXECUTE__=(
7882
'input/subtotals.csv',
7983
'output/totals.csv',
80-
'error/test_reporting_command.log')
81-
)
84+
'log/test_reporting_command_in_isolation.log'))
8285
return
8386

84-
def test_streaming_command(self, m):
87+
def test_reporting_command_on_server(self):
88+
pass
89+
90+
def test_streaming_command_in_isolation(self):
8591
self._run(
8692
'countmatches', [
8793
'fieldname=word_count',
8894
'pattern=\\w+',
8995
'text'],
9096
__GETINFO__=(
9197
'input/tweets.csv',
92-
'output/tweet_and_word_counts.csv',
93-
'error/test_streaming_command.log'),
98+
'output/tweets_with_word_count.csv',
99+
'log/test_streaming_command.log'),
94100
__EXECUTE__=(
95101
'input/tweets.csv',
96-
'output/tweet_and_word_counts.csv',
97-
'error/test_generating_command.log')
98-
)
102+
'output/tweets_with_word_count.csv',
103+
'log/test_generating_command_in_isolation.log'))
99104
return
100105

106+
def test_streaming_command_on_server(self):
107+
pass
108+
101109
def _run(self, command, args, **kwargs):
102110
for operation in ['__GETINFO__', '__EXECUTE__']:
103111
files = kwargs[operation]
@@ -127,5 +135,7 @@ def _start_process(cls, args, stdin, stdout, stderr):
127135
data_directory = os.path.join(package_directory, 'searchcommands_data')
128136
app_bin = os.path.join(os.path.dirname(package_directory), "examples/searchcommands_app/bin")
129137

138+
_seed = '\xcd{\xf8\xc4\x1c8=\x88\nc\xe2\xc4\xee\xdb\xcal'
139+
130140
if __name__ == "__main__":
131141
unittest.main()

0 commit comments

Comments
 (0)