Merge branch 'master' of https://github.com/Chaffelson/nipyapi into next

Chaffelson · Chaffelson · commit ebe3a0ceecca · 2020-02-01T17:42:37.000Z
� Conflicts:
�	nipyapi/canvas.py
�	nipyapi/versioning.py
�	resources/docker/tox-full/docker-compose.yml
�	tests/test_canvas.py
diff --git a/docs/devnotes.rst b/docs/devnotes.rst
@@ -14,6 +14,13 @@ Decision Points
 * We use Google style Docstrings to better enable Sphinx to produce nicely readable documentation
 
 
+Testing Notes
+-------------
+
+When running tests on new code, you are advised to run 'test_default' first, then 'test_regression', then finally 'test_security'.
+Because of the way errors are propagated you may have code failures which cause a teardown which then fails because of security controls, which can then obscure the original error.
+
+
 Docker Test Environment
 -----------------------
 
@@ -23,6 +30,30 @@ There is an Apache NiFi image available on Dockerhub::
 
 There are a couple of configuration files for launching various Docker environment configurations in ./test_env_config for convenience.
 
+Remote Testing on Centos7
+-------------------------
+
+Deploy a 4x16 or better on EC2 running Centos 7.5 or better, ssh in as root::
+
+    yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+    yum update -y
+    yum install -y centos-release-scl yum-utils device-mapper-persistent-data lvm2
+    yum install -y rh-python36 docker
+    systemctl start docker
+    scl enable rh-python36 bash
+    sudo curl -L "https://github.com/docker/compose/releases/download/1.25.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+    sudo chmod +x /usr/local/bin/docker-compose
+    sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+
+Set up remote execution environment to this server from your IDE, such as PyCharm.
+Python3 will be in a path like /opt/rh/rh-python36/root/usr/bin/python
+These commands are conveniently presented in /resources/test_setup/setup_centos7.sh
+
+You will then want to open up /home/centos/tmp/<pycharmprojectname>/resources/docker/tox-full and run::
+
+    docker-compose pull
+    docker-compose up -d
+
 Testing on OSX
 --------------
 
diff --git a/nipyapi/canvas.py b/nipyapi/canvas.py
@@ -1338,14 +1338,16 @@ def get_remote_process_group(rpg_id, summary=False):
     return out
 
 
-def create_remote_process_group(target_uris, transport='RAW', pg_id='root', position=None):
+def create_remote_process_group(target_uris, transport='RAW', pg_id='root',
+                                position=None):
     """
     Creates a new Remote Process Group with given parameters
 
     Args:
         target_uris (str): Comma separated list of target URIs
         transport (str): optional, RAW or HTTP
-        pg_id (str): optional, UUID of parent Process Group for remote process group
+        pg_id (str): optional, UUID of parent Process Group for remote
+          process group
         position (tuple): optional, tuple of location ints
 
     Returns:
@@ -1388,18 +1390,21 @@ def delete_remote_process_group(rpg, refresh=True):
     assert isinstance(rpg, nipyapi.nifi.RemoteProcessGroupEntity)
     if refresh:
         rpg = get_remote_process_group(rpg.id)
+    handle = nipyapi.nifi.RemoteProcessGroupsApi()
     with nipyapi.utils.rest_exceptions():
-        return nipyapi.nifi.RemoteProcessGroupsApi().remove_remote_process_group(
+        return handle.remove_remote_process_group(
             id=rpg.id,
             version=rpg.revision.version
         )
 
 
 def set_remote_process_group_transmission(rpg, enable=True, refresh=True):
     """
+    Enable or Disable Transmission for an RPG
 
     Args:
-        rpg (RemoteProcessGroupEntity): The ID of the remote process group to modify
+        rpg (RemoteProcessGroupEntity): The ID of the remote process group
+          to modify
         enable (bool): True to enable, False to disable
         refresh (bool): Whether to refresh the object before action
 
@@ -1410,8 +1415,9 @@ def set_remote_process_group_transmission(rpg, enable=True, refresh=True):
     assert isinstance(enable, bool)
     if refresh:
         rpg = get_remote_process_group(rpg.id)
+    handle = nipyapi.nifi.RemoteProcessGroupsApi()
     with nipyapi.utils.rest_exceptions():
-        return nipyapi.nifi.RemoteProcessGroupsApi().update_remote_process_group_run_status(
+        return handle.update_remote_process_group_run_status(
             id=rpg.id,
             body=nipyapi.nifi.RemotePortRunStatusEntity(
                 state='TRANSMITTING' if enable else 'STOPPED',
@@ -1482,6 +1488,7 @@ def get_funnel(funnel_id):
 
 def create_funnel(pg_id, position=None):
     """
+    Creates a Funnel Object
 
     Args:
         pg_id (str): ID of the parent Process Group
@@ -1510,6 +1517,7 @@ def create_funnel(pg_id, position=None):
 
 def delete_funnel(funnel, refresh=True):
     """
+    Deletes a Funnel Object
 
     Args:
         funnel (FunnelEntity): The Funnel to delete
diff --git a/nipyapi/security.py b/nipyapi/security.py
@@ -145,7 +145,7 @@ def create_service_user_group(identity, service='nifi',
     Args:
         identity (str): Identity string for the user group
         service (str): 'nifi' or 'registry'
-        users (list): A list of UserEntities belonging to the group
+        users (list): A list of nifi.UserEntity or registry.User belonging to the group
         strict (bool): Whether to throw an error on already exists
 
     Returns:
@@ -751,7 +751,9 @@ def bootstrap_security_policies(service, user_identity=None,
         None
 
     """
-    assert service in _valid_services
+    assert service in _valid_services, "service not in %s" % _valid_services
+    if user_identity is not None:
+        assert user_identity in [nipyapi.nifi.UserEntity, nipyapi.registry.User]
     if 'nifi' in service:
         rpg_id = nipyapi.canvas.get_root_pg_id()
         if user_identity is None and group_identity is None:
diff --git a/nipyapi/templates.py b/nipyapi/templates.py
@@ -46,14 +46,15 @@ def get_template_by_name(name):
     return None
 
 
-def get_template(identifier, identifier_type='name'):
+def get_template(identifier, identifier_type='name', greedy=False):
     """
     Filters the list of all Templates for a given string in a given field.
     Note that filters are configured in config.py
 
     Args:
         identifier (str): The string to filter on
         identifier_type (str): The identifier of the field to filter on
+        greedy (bool): True for greedy match, False for exact match
 
     Returns:
         None for no matches, Single Object for unique match,
@@ -65,7 +66,8 @@ def get_template(identifier, identifier_type='name'):
     with nipyapi.utils.rest_exceptions():
         obj = nipyapi.templates.list_all_templates(native=False)
     if obj:
-        return nipyapi.utils.filter_obj(obj, identifier, identifier_type)
+        return nipyapi.utils.filter_obj(obj, identifier, identifier_type,
+                                        greedy)
     return obj
 
 
diff --git a/nipyapi/utils.py b/nipyapi/utils.py
@@ -476,6 +476,8 @@ def infer_object_label_from_class(obj):
         return 'FUNNEL'
     if isinstance(obj, nipyapi.nifi.PortEntity):
         return obj.port_type
+    if isinstance(obj, nipyapi.nifi.RemoteProcessGroupDTO):
+        return 'REMOTEPROCESSGROUP'
     if isinstance(obj, nipyapi.nifi.RemoteProcessGroupPortDTO):
         # get RPG summary, find id of obj in input or output list
         parent_rpg = nipyapi.canvas.get_remote_process_group(
@@ -513,6 +515,7 @@ def bypass_slash_encoding(service, bypass):
 
 @contextmanager
 def rest_exceptions():
+    """Simple exception wrapper for Rest Exceptions"""
     try:
         yield
     except (nipyapi.nifi.rest.ApiException,
diff --git a/nipyapi/versioning.py b/nipyapi/versioning.py
@@ -210,7 +210,8 @@ def get_flow_in_bucket(bucket_id, identifier, identifier_type='name'):
 
 
 def save_flow_ver(process_group, registry_client, bucket, flow_name=None,
-                  flow_id=None, comment='', desc='', refresh=True):
+                  flow_id=None, comment='', desc='', refresh=True,
+                  force=False):
     """
     Adds a Process Group into NiFi Registry Version Control, or saves a new
     version to an existing VersionedFlow with a new version
@@ -228,7 +229,8 @@ def save_flow_ver(process_group, registry_client, bucket, flow_name=None,
             the bucket, if saving a new version to an existing flow
         comment (str): A comment for the version commit
         desc (str): A description of the VersionedFlow
-        refresh (bool): whether to refresh the object revisions before action
+        refresh (bool): Whether to refresh the object revisions before action
+        force (bool): Whether to Force Commit, or just regular Commit
 
     Returns:
         (VersionControlInformationEntity)
@@ -237,21 +239,36 @@ def save_flow_ver(process_group, registry_client, bucket, flow_name=None,
         target_pg = nipyapi.canvas.get_process_group(process_group.id, 'id')
     else:
         target_pg = process_group
+    if nipyapi.utils.check_version('1.10.0') <= 0:
+        body = nipyapi.nifi.StartVersionControlRequestEntity(
+            process_group_revision=target_pg.revision,
+            versioned_flow=nipyapi.nifi.VersionedFlowDTO(
+                bucket_id=bucket.identifier,
+                comments=comment,
+                description=desc,
+                flow_name=flow_name,
+                flow_id=flow_id,
+                registry_id=registry_client.id,
+                action='FORCE_COMMIT' if force else 'COMMIT'
+            )
+        )
+    else:
+        # no 'action' property in versions < 1.10
+        body = nipyapi.nifi.StartVersionControlRequestEntity(
+            process_group_revision=target_pg.revision,
+            versioned_flow={
+                'bucketId': bucket.identifier,
+                'comments': comment,
+                'description': desc,
+                'flowName': flow_name,
+                'flowId': flow_id,
+                'registryId': registry_client.id
+            }
+        )
     with nipyapi.utils.rest_exceptions():
         return nipyapi.nifi.VersionsApi().save_to_flow_registry(
             id=target_pg.id,
-            body=nipyapi.nifi.StartVersionControlRequestEntity(
-                process_group_revision=target_pg.revision,
-                versioned_flow=nipyapi.nifi.VersionedFlowDTO(
-                    bucket_id=bucket.identifier,
-                    comments=comment,
-                    description=desc,
-                    flow_name=flow_name,
-                    flow_id=flow_id,
-                    registry_id=registry_client.id,
-                    action='COMMIT'
-                )
-            )
+            body=body
         )
 
 
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -7,6 +7,7 @@ bumpversion>=0.5.3
 watchdog>=0.8.3
 twine>=1.9.1,<2.0.0  # pyup: ignore
 virtualenvwrapper>=4.8
+virtualenv>=16.0.0  # required for tox 3.14.2 but not forced
 
 # Testing
 tox>=2.9.1
@@ -28,4 +29,3 @@ cryptography>=2.1.2
 py>=1.4.31
 randomize>=0.13
 certifi>=2017.7.27.1
-virtualenvwrapper>=4.8
diff --git a/resources/docker/tox-full/docker-compose.yml b/resources/docker/tox-full/docker-compose.yml
@@ -18,6 +18,12 @@ services:
     hostname: nifi-180
     ports:
       - "10180:8080"
+  nifi-192:
+    image: apache/nifi:1.9.2
+    container_name: nifi-192
+    hostname: nifi-192
+    ports:
+      - "10192:8080"
   nifi:
     image: apache/nifi:1.10.0
     container_name: nifi
@@ -32,6 +38,14 @@ services:
       - "18010:18010"
     environment:
       - NIFI_REGISTRY_WEB_HTTP_PORT=18010
+  registry-030:
+    image: apache/nifi-registry:0.3.0
+    container_name: registry-030
+    hostname: registry-030
+    ports:
+      - "18030:18030"
+    environment:
+      - NIFI_REGISTRY_WEB_HTTP_PORT=18030
   registry:
     image: apache/nifi-registry:0.5.0
     container_name: registry
diff --git a/resources/test_setup/setup_centos7.sh b/resources/test_setup/setup_centos7.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+set -u
+
+sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+sudo yum update -y
+sudo yum install -y centos-release-scl yum-utils device-mapper-persistent-data lvm2
+sudo yum install -y rh-python36 docker
+sudo systemctl start docker
+sudo curl -L "https://github.com/docker/compose/releases/download/1.25.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+source /opt/rh/python36/enable
+pip install --upgrade pip
+pip install -r ../../requirements.txt
+pip install -r ../../requirements_dev.txts
+
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -43,6 +43,7 @@
 # Test template filenames should match the template PG name
 test_templates = {
     'basic': test_basename + 'Template_00',
+    'greedy': test_basename + 'Template_00_greedy',
     'complex': test_basename + 'Template_01'
 }
 
@@ -55,6 +56,7 @@
             'http://' + test_host + ':10112/nifi-api',
             'http://' + test_host + ':10120/nifi-api',
             'http://' + test_host + ':10180/nifi-api',
+            'http://' + test_host + ':10192/nifi-api',
         ]
 secure_nifi_endpoints = ['https://' + test_host + ':8443/nifi-api']
 default_registry_endpoints = [
@@ -67,6 +69,10 @@
             ('http://' + test_host + ':18010/nifi-registry-api',
                 'http://registry-010:18010',
              'http://' + test_host + ':8080/nifi-api'
+             ),
+            ('http://' + test_host + ':18030/nifi-registry-api',
+                'http://registry-030:18030',
+             'http://' + test_host + ':10192/nifi-api'
              )
         ]
 secure_registry_endpoints = [
@@ -389,7 +395,7 @@ def fixture_templates(request, fix_pg):
              nipyapi.config.nifi_config.host)
     FixtureTemplates = namedtuple(
         'FixtureTemplates', ('pg', 'b_file', 'b_name', 'c_file',
-                             'c_name')
+                             'c_name', 'g_name', 'g_file')
     )
     f_pg = fix_pg
     f_b_file = path.join(
@@ -404,11 +410,19 @@ def fixture_templates(request, fix_pg):
             test_templates['complex'] + '.xml'
         )
     f_c_name = 'nipyapi_testTemplate_01'
+    f_g_file = path.join(
+        path.dirname(__file__),
+        test_resource_dir,
+        test_templates['greedy'] + '.xml'
+    )
+    f_g_name = 'nipyapi_testTemplate_00_greedy'
     out = FixtureTemplates(
         pg=f_pg,
         b_name=f_b_name,
         c_name=f_c_name,
+        g_name=f_g_name,
         b_file=f_b_file,
+        g_file=f_g_file,
         c_file=f_c_file
     )
     request.addfinalizer(remove_test_templates)
diff --git a/tests/resources/nipyapi_testTemplate_00_greedy.xml b/tests/resources/nipyapi_testTemplate_00_greedy.xml
diff --git a/tests/test_canvas.py b/tests/test_canvas.py
diff --git a/tests/test_security.py b/tests/test_security.py
diff --git a/tests/test_templates.py b/tests/test_templates.py
