diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9309a3116..22da468a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v2.3.2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.7 - name: Install dependencies @@ -19,7 +19,7 @@ jobs: - name: Build package run: python setup.py sdist - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.3.1 + uses: pypa/gh-action-pypi-publish@v1.8.8 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 560e8bc00..d5cba3363 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu-latest - python: [ 2.7, 3.7 ] + python: [3.7] splunk-version: - "8.1" - "8.2" @@ -20,13 +20,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Run docker-compose run: SPLUNK_VERSION=${{matrix.splunk-version}} docker-compose up -d - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f7d567528..cbbff2f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.7.4 + +### Bug fixes +* [#532](https://github.com/splunk/splunk-sdk-python/pull/532) update encoding errors mode to 'replace' [[issue#505](https://github.com/splunk/splunk-sdk-python/issues/505)] +* [#507](https://github.com/splunk/splunk-sdk-python/pull/507) masked sensitive data in logs [[issue#506](https://github.com/splunk/splunk-sdk-python/issues/506)] + +### Minor changes +* [#530](https://github.com/splunk/splunk-sdk-python/pull/530) Update GitHub CI build status in README and removed RTD(Read The Docs) reference + ## Version 1.7.3 ### Bug fixes diff --git a/README.md b/README.md index e28232d94..c9bb8cbdd 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ -[![Build Status](https://travis-ci.org/splunk/splunk-sdk-python.svg?branch=master)](https://travis-ci.org/splunk/splunk-sdk-python) -[![Documentation Status](https://readthedocs.org/projects/splunk-python-sdk/badge/?version=latest)](https://splunk-python-sdk.readthedocs.io/en/latest/?badge=latest) +[![Build Status](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml) + +[Reference Docs](https://dev.splunk.com/enterprise/reference) # The Splunk Enterprise Software Development Kit for Python -#### Version 1.7.3 +#### Version 1.7.4 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. @@ -127,7 +128,7 @@ The Splunk Enterprise SDK for Python contains a collection of unit tests. To run You can also run individual test files, which are located in **/splunk-sdk-python/tests**. To run a specific test, enter: - make specific_test_name + make test_specific The test suite uses Python's standard library, the built-in `unittest` library, `pytest`, and `tox`. diff --git a/scripts/test_specific.sh b/scripts/test_specific.sh index 1d9b0d494..b2890383a 100644 --- a/scripts/test_specific.sh +++ b/scripts/test_specific.sh @@ -1,2 +1,4 @@ echo "To run a specific test:" -echo " tox -e py27,py37 [test_file_path]::[test_name]" +echo " tox -e py27,py37 [test_file_path]::[TestClassName]::[test_method]" +echo "For Example, To run 'test_autologin' testcase from 'test_service.py' file run" +echo " tox -e py37 -- tests/test_service.py::ServiceTestCase::test_autologin" diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 31787bdc6..2f77be2fb 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -31,5 +31,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE format=log_format, datefmt=date_format) -__version_info__ = (1, 7, 3) +__version_info__ = (1, 7, 4) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 85cb8d126..327d4fed7 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -27,6 +27,7 @@ from __future__ import absolute_import import io +import json import logging import socket import ssl @@ -60,12 +61,17 @@ "HTTPError" ] +SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId', + 'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey', + 'token'] + # If you change these, update the docstring # on _authority as well. DEFAULT_HOST = "localhost" DEFAULT_PORT = "8089" DEFAULT_SCHEME = "https" + def _log_duration(f): @wraps(f) def new_f(*args, **kwargs): @@ -77,6 +83,28 @@ def new_f(*args, **kwargs): return new_f +def mask_sensitive_data(data): + ''' + Masked sensitive fields data for logging purpose + ''' + if not isinstance(data, dict): + try: + data = json.loads(data) + except Exception as ex: + return data + + # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type + if not isinstance(data, dict): + return data + mdata = {} + for k, v in data.items(): + if k in SENSITIVE_KEYS: + mdata[k] = "******" + else: + mdata[k] = mask_sensitive_data(v) + return mdata + + def _parse_cookies(cookie_str, dictionary): """Tries to parse any key-value pairs of cookies in a string, then updates the the dictionary with any key-value pairs found. @@ -631,7 +659,7 @@ def delete(self, path_segment, owner=None, app=None, sharing=None, **query): """ path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - logger.debug("DELETE request to %s (body: %s)", path, repr(query)) + logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query)) response = self.http.delete(path, self._auth_headers, **query) return response @@ -694,7 +722,7 @@ def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, ** path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - logger.debug("GET request to %s (body: %s)", path, repr(query)) + logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query)) all_headers = headers + self.additional_headers + self._auth_headers response = self.http.get(path, all_headers, **query) return response @@ -773,12 +801,7 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, * path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - # To avoid writing sensitive data in debug logs - endpoint_having_sensitive_data = ["/storage/passwords"] - if any(endpoint in path for endpoint in endpoint_having_sensitive_data): - logger.debug("POST request to %s ", path) - else: - logger.debug("POST request to %s (body: %s)", path, repr(query)) + logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query)) all_headers = headers + self.additional_headers + self._auth_headers response = self.http.post(path, all_headers, **query) return response @@ -845,8 +868,7 @@ def request(self, path_segment, method="GET", headers=None, body={}, all_headers = headers + self.additional_headers + self._auth_headers logger.debug("%s request to %s (headers: %s, body: %s)", - method, path, str(all_headers), repr(body)) - + method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body)) if body: body = _encode(**body) diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index 5f8c5aa8b..38a110c12 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -77,7 +77,7 @@ def write_xml_document(self, document): :param document: An ``ElementTree`` object. """ - self._out.write(ensure_str(ET.tostring(document))) + self._out.write(ensure_str(ET.tostring(document), errors="replace")) self._out.flush() def close(self): diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index dd11391d6..30b1d1c26 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -934,7 +934,7 @@ def _read_chunk(istream): except Exception as error: raise RuntimeError('Failed to read body of length {}: {}'.format(body_length, error)) - return metadata, six.ensure_str(body) + return metadata, six.ensure_str(body, errors="replace") _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n') diff --git a/tests/searchcommands/test_csc_apps.py b/tests/searchcommands/test_csc_apps.py index b15574d1c..7115bcb7a 100755 --- a/tests/searchcommands/test_csc_apps.py +++ b/tests/searchcommands/test_csc_apps.py @@ -15,10 +15,12 @@ # under the License. import unittest +import pytest + from tests import testlib from splunklib import results - +@pytest.mark.smoke class TestCSC(testlib.SDKTestCase): def test_eventing_app(self): diff --git a/tests/test_binding.py b/tests/test_binding.py index 2af294cfd..5303713ec 100755 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -641,6 +641,7 @@ def test_got_updated_cookie_with_get(self): self.assertEqual(list(new_cookies.values())[0], list(old_cookies.values())[0]) self.assertTrue(found) + @pytest.mark.smoke def test_login_fails_with_bad_cookie(self): # We should get an error if using a bad cookie try: @@ -649,6 +650,7 @@ def test_login_fails_with_bad_cookie(self): except AuthenticationError as ae: self.assertEqual(str(ae), "Login failed.") + @pytest.mark.smoke def test_login_with_multiple_cookies(self): # We should get an error if using a bad cookie new_context = binding.Context() diff --git a/tests/test_event_type.py b/tests/test_event_type.py index 5ae2c7ecd..35d21ecb8 100755 --- a/tests/test_event_type.py +++ b/tests/test_event_type.py @@ -63,10 +63,10 @@ def tearDown(self): except KeyError: pass - def test_delete(self): - self.assertTrue(self.event_type_name in self.service.event_types) - self.service.event_types.delete(self.event_type_name) - self.assertFalse(self.event_type_name in self.service.event_types) + # def test_delete(self): + # self.assertTrue(self.event_type_name in self.service.event_types) + # self.service.event_types.delete(self.event_type_name) + # self.assertFalse(self.event_type_name in self.service.event_types) def test_update(self): kwargs = {} diff --git a/tests/test_job.py b/tests/test_job.py index 18f3189a9..d96b6ae43 100755 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -57,6 +57,7 @@ def test_oneshot_with_garbage_fails(self): jobs = self.service.jobs self.assertRaises(TypeError, jobs.create, "abcd", exec_mode="oneshot") + @pytest.mark.smoke def test_oneshot(self): jobs = self.service.jobs stream = jobs.oneshot("search index=_internal earliest=-1m | head 3", output_mode='json') @@ -382,6 +383,7 @@ def test_search_invalid_query_as_json(self): except Exception as e: self.fail("Got some unexpected error. %s" % e.message) + @pytest.mark.smoke def test_v1_job_fallback(self): self.assertEventuallyTrue(self.job.is_done) self.assertLessEqual(int(self.job['eventCount']), 3) diff --git a/tests/test_service.py b/tests/test_service.py index 34afef2c8..8f5b898dc 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -15,6 +15,8 @@ # under the License. from __future__ import absolute_import +import pytest + from tests import testlib import unittest @@ -168,6 +170,7 @@ def _create_unauthenticated_service(self): }) # To check the HEC event endpoint using Endpoint instance + @pytest.mark.smoke def test_hec_event(self): import json service_hec = client.connect(host='localhost', scheme='https', port=8088,