From 9b4c59d74533e6c00417f80e3c4d22cb5963bdcf Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Sun, 15 Oct 2023 17:43:27 +0900 Subject: [PATCH 01/15] feat: add allow_disk_use option to analyze query (cherry picked from commit 5972549652c4739f3dd519b6d79d480200fe52f8) --- src/spaceone/core/model/mongo_model/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/spaceone/core/model/mongo_model/__init__.py b/src/spaceone/core/model/mongo_model/__init__.py index 0062009..e8c5a1b 100644 --- a/src/spaceone/core/model/mongo_model/__init__.py +++ b/src/spaceone/core/model/mongo_model/__init__.py @@ -1309,7 +1309,8 @@ def _convert_date_value(cls, date_value, date_field_format): @classmethod def analyze(cls, *args, granularity=None, fields=None, select=None, group_by=None, field_group=None, filter=None, filter_or=None, page=None, sort=None, start=None, end=None, - date_field='date', date_field_format='%Y-%m-%d', target='SECONDARY_PREFERRED', **kwargs): + date_field='date', date_field_format='%Y-%m-%d', target='SECONDARY_PREFERRED', + allow_disk_use=False, **kwargs): if fields is None: raise ERROR_REQUIRED_PARAMETER(key='fields') @@ -1348,7 +1349,8 @@ def analyze(cls, *args, granularity=None, fields=None, select=None, group_by=Non } } ], - 'target': target + 'target': target, + 'allow_disk_use': allow_disk_use } if select: From 1359cc9f0bd40ac3fe1f78bced0843d0172f55ca Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Wed, 30 Aug 2023 14:41:53 +0900 Subject: [PATCH 02/15] feat: add override openapi step when start fastapi application Signed-off-by: ImMin5 (cherry picked from commit c8c799dfc179e74d51def82a6af33d1ad5d14d32) --- src/spaceone/core/fastapi/server.py | 101 ++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index 7066161..701f3f4 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -1,8 +1,12 @@ import logging import uvicorn +import json +import os from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.docs import get_swagger_ui_html +from fastapi.responses import HTMLResponse from spaceone.core import config from spaceone.core.logger import set_logger @@ -130,10 +134,107 @@ def _init_fast_api(): ) +def _get_all_services_from_openapi_json_files(openapi_json_path): + services = [] + openapi_json_files = os.listdir(openapi_json_path) + for openapi_json_file in openapi_json_files: + services.append('_'.join(openapi_json_file.split('_')[:-1]).lower()) + return services + + +def _sort_services(services): + return sorted(services, key=lambda x: ('identity' not in x, 'inventory' not in x, 'cost-analysis' not in x, + 'monitoring' not in x, 'notification' not in x, 'repository' not in x, x)) + + +def _create_openapi_json(app, service_name): + swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') + swagger_path = os.path.join(swagger_path, f"{service_name.replace('-', '_')}_openapi.json") + try: + with open(swagger_path, 'r') as f: + custom_openapi_schema = json.loads(f.read()) + custom_openapi_schema['openapi'] = app.openapi().get('openapi') + description = custom_openapi_schema['info']['summary'] + app.openapi()['info']['description'] += f"| **{service_name.replace('-', ' ').title()}** | {description} | [/{service_name}/docs](/{service_name}/docs) |\n" + + with open(swagger_path, 'w') as f: + json.dump(custom_openapi_schema, f, indent=2) + except Exception as e: + _LOGGER.error(f'[_create_openapi_json] {swagger_path} : {e}', exc_info=True) + + +def _override_openapi(app): + extension_swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') + if not os.path.exists(extension_swagger_path): + _LOGGER.info(f'[_override_openapi] Extension Swagger Path is not exists. (path = {extension_swagger_path})') + return app + + services = _get_all_services_from_openapi_json_files(extension_swagger_path) + services = _sort_services(services) + _openapi_info = app.openapi().get('info') + _openapi_version = app.openapi().get('openapi') + + app.openapi()['info']['description'] += "\n

\n" + app.openapi()['info']['description'] += "\n## List of Services\n" + app.openapi()['info']['description'] += "\n[Home](/docs)\n" + app.openapi()['info']['description'] += "| **Service** | **Description** | **URL** |\n" + app.openapi()['info']['description'] += "|:---|:--- |:---|\n" + + for service in services: + service = service.replace('_', '-') + _create_openapi_json(app, service_name=service) + build_docs(app, prefix=f"/{service}", service_name=service) + + app.openapi()['info'][ + 'description'] += "| **Console API** | Service that offers features exclusive to the Console API. | [/docs](/docs#console-api%20%3E%20api) |\n" + + return app + + +def build_docs( + app: FastAPI, + prefix: str, + service_name: str +) -> None: + async def get_openapi(): + swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') + swagger_path = os.path.join(swagger_path, f"{service_name.replace('-','_')}_openapi.json") + with open(swagger_path, 'r') as f: + custom_openapi_schema = json.loads(f.read()) + return custom_openapi_schema + + get_openapi.__name__ = get_openapi.__name__ + prefix + app.add_api_route(prefix + "/openapi.json", get_openapi, include_in_schema=False) + + async def swagger_ui_html() -> HTMLResponse: + return get_swagger_ui_html( + openapi_url=prefix + "/openapi.json", + title=f'{service_name.title().replace("-"," ")} API' + ' - Swagger UI', + oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, + init_oauth=app.swagger_ui_init_oauth, + ) + + swagger_ui_html.__name__ = swagger_ui_html.__name__ + prefix + app.add_api_route(prefix + "/docs", swagger_ui_html, include_in_schema=False) + + +def _get_all_services_list(app): + services = [] + for route in app.routes: + path = route.path.split('/') + if len(path) == 4: + services.append(path[1].replace('-', '_')) + + services = list(set(services)) + sorted_services = sorted(services, key=lambda x: ('identity' not in x, 'inventory' not in x, 'cost_analysis' not in x, x)) + return sorted_services + + def fast_api_app(): app = _init_fast_api() app = _add_middlewares(app) app = _include_routers(app) + app = _override_openapi(app) return app From d4347f3363126ec36df7683deac89492bb0adf06 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Wed, 30 Aug 2023 14:53:36 +0900 Subject: [PATCH 03/15] feat: remove default extension.reflection router Signed-off-by: ImMin5 (cherry picked from commit 6622ad345b6980ab52ca0d5eddf2dbd5448d3934) --- src/spaceone/core/config/default_conf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/spaceone/core/config/default_conf.py b/src/spaceone/core/config/default_conf.py index 0dda8e9..92c338f 100644 --- a/src/spaceone/core/config/default_conf.py +++ b/src/spaceone/core/config/default_conf.py @@ -36,10 +36,6 @@ { 'router_path': 'spaceone.core.fastapi.extension.health:router', 'router_options': {} - }, - { - 'router_path': 'spaceone.core.fastapi.extension.reflection:router', - 'router_options': {} } ] From 1b09ba7f88282694097a4f77b856a7f8fe1d3431 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Thu, 31 Aug 2023 18:10:47 +0900 Subject: [PATCH 04/15] feat: separate a function for adding external Swagger to an openapi.py Signed-off-by: ImMin5 (cherry picked from commit 2caf705fd3531480ec95b057565e5f97826ae3d5) --- src/spaceone/core/fastapi/openapi.py | 122 +++++++++++++++++++++++++++ src/spaceone/core/fastapi/server.py | 105 +---------------------- 2 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 src/spaceone/core/fastapi/openapi.py diff --git a/src/spaceone/core/fastapi/openapi.py b/src/spaceone/core/fastapi/openapi.py new file mode 100644 index 0000000..0594f34 --- /dev/null +++ b/src/spaceone/core/fastapi/openapi.py @@ -0,0 +1,122 @@ +import logging +import json +import os + +from fastapi import FastAPI +from fastapi.openapi.docs import get_swagger_ui_html +from fastapi.responses import HTMLResponse + +from spaceone.core import config + +_LOGGER = logging.getLogger(__name__) + + +def _add_external_api_route( + app: FastAPI, + prefix: str, + service_name: str, + external_swagger_path: str +) -> None: + async def get_openapi(): + openapi_json_file = os.path.join(external_swagger_path, f"{service_name}_openapi.json") + with open(openapi_json_file, 'r') as f: + custom_openapi_schema = json.loads(f.read()) + return custom_openapi_schema + + get_openapi.__name__ = get_openapi.__name__ + prefix + app.add_api_route(prefix + "/openapi.json", get_openapi, include_in_schema=False) + + async def swagger_ui_html() -> HTMLResponse: + return get_swagger_ui_html( + openapi_url=prefix + "/openapi.json", + title=f'{service_name.title().replace("-", " ")} API' + ' - Swagger UI', + oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, + init_oauth=app.swagger_ui_init_oauth, + ) + + swagger_ui_html.__name__ = swagger_ui_html.__name__ + prefix + app.add_api_route(prefix + "/docs", swagger_ui_html, include_in_schema=False) + + +def _services_from_openapi_json_files(openapi_json_path): + """ + openapi json file must follow {service}_openapi.json file name format. + if '-' exist in {service} converted to '_'. + """ + services = [] + openapi_json_files = os.listdir(openapi_json_path) + for openapi_json_file in openapi_json_files: + services.append('_'.join(openapi_json_file.split('_')[:-1]).lower()) + return services + + +def _sort_services(services): + external_swagger_priority_services = config.get_global('EXTERNAL_SWAGGER_PRIORITY_SERVICES', []) + external_swagger_priority_services = [external_service.replace('-', '_').lower() for external_service in + external_swagger_priority_services] + + sorted_services = list(dict.fromkeys(external_swagger_priority_services + sorted(services))) + return sorted_services + + +def _create_openapi_json(app, service_name, external_swagger_path, service_description): + openapi_json_file = os.path.join(external_swagger_path, f"{service_name}_openapi.json") + service_name_with_dash = service_name.replace('_', '-') + try: + with open(openapi_json_file, 'r') as f: + custom_openapi_schema = json.loads(f.read()) + custom_openapi_schema['openapi'] = app.openapi().get('openapi') + description = custom_openapi_schema['info']['summary'] + service_description[service_name] = f"| **{service_name.replace('_', ' ').title()}** | {description} | [/{service_name_with_dash}/docs](/{service_name_with_dash}/docs) |\n" + + with open(openapi_json_file, 'w') as f: + json.dump(custom_openapi_schema, f, indent=2) + except Exception as e: + _LOGGER.error(f'[_create_openapi_json] {openapi_json_file} : {e}', exc_info=True) + + +def _check_external_swagger_path(external_swagger_path): + if not external_swagger_path: + _LOGGER.info('[_check_external_swagger_path] EXTERNAL_SWAGGER_PATH is not set') + return False + + if not os.path.exists(external_swagger_path): + _LOGGER.error(f'[_check_external_swagger_path] "{external_swagger_path}" : Not Found') + return False + + return True + + +def _create_external_apis_description(app, service_description): + app.openapi()['info']['description'] += "\n

\n" + app.openapi()['info']['description'] += "\n## List of External APIs\n" + app.openapi()['info']['description'] += "\n[Home](/docs)\n" + app.openapi()['info']['description'] += "| **Name** | **Description** | **URL** |\n" + app.openapi()['info']['description'] += "|:---|:--- |:---|\n" + app.openapi()['info']['description'] += ''.join(service_description.values()) + + +def add_external_swagger(app): + try: + external_swagger_path = config.get_global('EXTERNAL_SWAGGER_PATH') + + if not _check_external_swagger_path(external_swagger_path): + return app + + services = _services_from_openapi_json_files(external_swagger_path) + sorted_services = _sort_services(services) + + service_description = {} + for service in sorted_services: + _create_openapi_json(app=app, service_name=service, external_swagger_path=external_swagger_path, + service_description=service_description) + _add_external_api_route(app=app, prefix=f"/{service.replace('_', '-')}", service_name=service, + external_swagger_path=external_swagger_path) + + if len(services) > 0: + _create_external_apis_description(app, service_description) + + except Exception as e: + _LOGGER.error(f'[add_external_swagger] {e}', exc_info=True) + finally: + return app diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index 701f3f4..e02faad 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -1,17 +1,14 @@ import logging import uvicorn -import json -import os -from fastapi import FastAPI, Request +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.openapi.docs import get_swagger_ui_html -from fastapi.responses import HTMLResponse from spaceone.core import config from spaceone.core.logger import set_logger from spaceone.core.opentelemetry import set_tracer, set_metric from spaceone.core.extension.server_info import ServerInfoManager +from spaceone.core.fastapi.openapi import add_external_swagger _LOGGER = logging.getLogger(__name__) @@ -134,107 +131,11 @@ def _init_fast_api(): ) -def _get_all_services_from_openapi_json_files(openapi_json_path): - services = [] - openapi_json_files = os.listdir(openapi_json_path) - for openapi_json_file in openapi_json_files: - services.append('_'.join(openapi_json_file.split('_')[:-1]).lower()) - return services - - -def _sort_services(services): - return sorted(services, key=lambda x: ('identity' not in x, 'inventory' not in x, 'cost-analysis' not in x, - 'monitoring' not in x, 'notification' not in x, 'repository' not in x, x)) - - -def _create_openapi_json(app, service_name): - swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') - swagger_path = os.path.join(swagger_path, f"{service_name.replace('-', '_')}_openapi.json") - try: - with open(swagger_path, 'r') as f: - custom_openapi_schema = json.loads(f.read()) - custom_openapi_schema['openapi'] = app.openapi().get('openapi') - description = custom_openapi_schema['info']['summary'] - app.openapi()['info']['description'] += f"| **{service_name.replace('-', ' ').title()}** | {description} | [/{service_name}/docs](/{service_name}/docs) |\n" - - with open(swagger_path, 'w') as f: - json.dump(custom_openapi_schema, f, indent=2) - except Exception as e: - _LOGGER.error(f'[_create_openapi_json] {swagger_path} : {e}', exc_info=True) - - -def _override_openapi(app): - extension_swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') - if not os.path.exists(extension_swagger_path): - _LOGGER.info(f'[_override_openapi] Extension Swagger Path is not exists. (path = {extension_swagger_path})') - return app - - services = _get_all_services_from_openapi_json_files(extension_swagger_path) - services = _sort_services(services) - _openapi_info = app.openapi().get('info') - _openapi_version = app.openapi().get('openapi') - - app.openapi()['info']['description'] += "\n

\n" - app.openapi()['info']['description'] += "\n## List of Services\n" - app.openapi()['info']['description'] += "\n[Home](/docs)\n" - app.openapi()['info']['description'] += "| **Service** | **Description** | **URL** |\n" - app.openapi()['info']['description'] += "|:---|:--- |:---|\n" - - for service in services: - service = service.replace('_', '-') - _create_openapi_json(app, service_name=service) - build_docs(app, prefix=f"/{service}", service_name=service) - - app.openapi()['info'][ - 'description'] += "| **Console API** | Service that offers features exclusive to the Console API. | [/docs](/docs#console-api%20%3E%20api) |\n" - - return app - - -def build_docs( - app: FastAPI, - prefix: str, - service_name: str -) -> None: - async def get_openapi(): - swagger_path = config.get_global('EXTENSION_SWAGGER_PATH') - swagger_path = os.path.join(swagger_path, f"{service_name.replace('-','_')}_openapi.json") - with open(swagger_path, 'r') as f: - custom_openapi_schema = json.loads(f.read()) - return custom_openapi_schema - - get_openapi.__name__ = get_openapi.__name__ + prefix - app.add_api_route(prefix + "/openapi.json", get_openapi, include_in_schema=False) - - async def swagger_ui_html() -> HTMLResponse: - return get_swagger_ui_html( - openapi_url=prefix + "/openapi.json", - title=f'{service_name.title().replace("-"," ")} API' + ' - Swagger UI', - oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, - init_oauth=app.swagger_ui_init_oauth, - ) - - swagger_ui_html.__name__ = swagger_ui_html.__name__ + prefix - app.add_api_route(prefix + "/docs", swagger_ui_html, include_in_schema=False) - - -def _get_all_services_list(app): - services = [] - for route in app.routes: - path = route.path.split('/') - if len(path) == 4: - services.append(path[1].replace('-', '_')) - - services = list(set(services)) - sorted_services = sorted(services, key=lambda x: ('identity' not in x, 'inventory' not in x, 'cost_analysis' not in x, x)) - return sorted_services - - def fast_api_app(): app = _init_fast_api() app = _add_middlewares(app) app = _include_routers(app) - app = _override_openapi(app) + app = add_external_swagger(app) return app From d0ef9908cff12742d14469743d71168b45d692be Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 1 Sep 2023 17:31:55 +0900 Subject: [PATCH 05/15] feat: remove openapi.py Signed-off-by: ImMin5 (cherry picked from commit 8a191aad7b89dc67227a60e51f76efca2f0a9d98) --- src/spaceone/core/fastapi/openapi.py | 122 --------------------------- 1 file changed, 122 deletions(-) delete mode 100644 src/spaceone/core/fastapi/openapi.py diff --git a/src/spaceone/core/fastapi/openapi.py b/src/spaceone/core/fastapi/openapi.py deleted file mode 100644 index 0594f34..0000000 --- a/src/spaceone/core/fastapi/openapi.py +++ /dev/null @@ -1,122 +0,0 @@ -import logging -import json -import os - -from fastapi import FastAPI -from fastapi.openapi.docs import get_swagger_ui_html -from fastapi.responses import HTMLResponse - -from spaceone.core import config - -_LOGGER = logging.getLogger(__name__) - - -def _add_external_api_route( - app: FastAPI, - prefix: str, - service_name: str, - external_swagger_path: str -) -> None: - async def get_openapi(): - openapi_json_file = os.path.join(external_swagger_path, f"{service_name}_openapi.json") - with open(openapi_json_file, 'r') as f: - custom_openapi_schema = json.loads(f.read()) - return custom_openapi_schema - - get_openapi.__name__ = get_openapi.__name__ + prefix - app.add_api_route(prefix + "/openapi.json", get_openapi, include_in_schema=False) - - async def swagger_ui_html() -> HTMLResponse: - return get_swagger_ui_html( - openapi_url=prefix + "/openapi.json", - title=f'{service_name.title().replace("-", " ")} API' + ' - Swagger UI', - oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, - init_oauth=app.swagger_ui_init_oauth, - ) - - swagger_ui_html.__name__ = swagger_ui_html.__name__ + prefix - app.add_api_route(prefix + "/docs", swagger_ui_html, include_in_schema=False) - - -def _services_from_openapi_json_files(openapi_json_path): - """ - openapi json file must follow {service}_openapi.json file name format. - if '-' exist in {service} converted to '_'. - """ - services = [] - openapi_json_files = os.listdir(openapi_json_path) - for openapi_json_file in openapi_json_files: - services.append('_'.join(openapi_json_file.split('_')[:-1]).lower()) - return services - - -def _sort_services(services): - external_swagger_priority_services = config.get_global('EXTERNAL_SWAGGER_PRIORITY_SERVICES', []) - external_swagger_priority_services = [external_service.replace('-', '_').lower() for external_service in - external_swagger_priority_services] - - sorted_services = list(dict.fromkeys(external_swagger_priority_services + sorted(services))) - return sorted_services - - -def _create_openapi_json(app, service_name, external_swagger_path, service_description): - openapi_json_file = os.path.join(external_swagger_path, f"{service_name}_openapi.json") - service_name_with_dash = service_name.replace('_', '-') - try: - with open(openapi_json_file, 'r') as f: - custom_openapi_schema = json.loads(f.read()) - custom_openapi_schema['openapi'] = app.openapi().get('openapi') - description = custom_openapi_schema['info']['summary'] - service_description[service_name] = f"| **{service_name.replace('_', ' ').title()}** | {description} | [/{service_name_with_dash}/docs](/{service_name_with_dash}/docs) |\n" - - with open(openapi_json_file, 'w') as f: - json.dump(custom_openapi_schema, f, indent=2) - except Exception as e: - _LOGGER.error(f'[_create_openapi_json] {openapi_json_file} : {e}', exc_info=True) - - -def _check_external_swagger_path(external_swagger_path): - if not external_swagger_path: - _LOGGER.info('[_check_external_swagger_path] EXTERNAL_SWAGGER_PATH is not set') - return False - - if not os.path.exists(external_swagger_path): - _LOGGER.error(f'[_check_external_swagger_path] "{external_swagger_path}" : Not Found') - return False - - return True - - -def _create_external_apis_description(app, service_description): - app.openapi()['info']['description'] += "\n

\n" - app.openapi()['info']['description'] += "\n## List of External APIs\n" - app.openapi()['info']['description'] += "\n[Home](/docs)\n" - app.openapi()['info']['description'] += "| **Name** | **Description** | **URL** |\n" - app.openapi()['info']['description'] += "|:---|:--- |:---|\n" - app.openapi()['info']['description'] += ''.join(service_description.values()) - - -def add_external_swagger(app): - try: - external_swagger_path = config.get_global('EXTERNAL_SWAGGER_PATH') - - if not _check_external_swagger_path(external_swagger_path): - return app - - services = _services_from_openapi_json_files(external_swagger_path) - sorted_services = _sort_services(services) - - service_description = {} - for service in sorted_services: - _create_openapi_json(app=app, service_name=service, external_swagger_path=external_swagger_path, - service_description=service_description) - _add_external_api_route(app=app, prefix=f"/{service.replace('_', '-')}", service_name=service, - external_swagger_path=external_swagger_path) - - if len(services) > 0: - _create_external_apis_description(app, service_description) - - except Exception as e: - _LOGGER.error(f'[add_external_swagger] {e}', exc_info=True) - finally: - return app From 9f0fd5f77ac1500b2a6c00b4d22ca0f01ca6e285 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 1 Sep 2023 17:32:55 +0900 Subject: [PATCH 06/15] feat: modify some of REST and gRPC config name Signed-off-by: ImMin5 (cherry picked from commit 93e5adc3ee9c0bdf5cfde176c8b6a64538c35622) --- src/spaceone/core/config/__init__.py | 4 ---- src/spaceone/core/config/default_conf.py | 29 +++++++++++++++--------- src/spaceone/core/fastapi/server.py | 8 +++---- src/spaceone/core/pygrpc/server.py | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/spaceone/core/config/__init__.py b/src/spaceone/core/config/__init__.py index b7429b5..6a55bae 100644 --- a/src/spaceone/core/config/__init__.py +++ b/src/spaceone/core/config/__init__.py @@ -37,10 +37,6 @@ def get_service(): return _GLOBAL['SERVICE'] -def get_extension_apis(): - return _GLOBAL.get('EXTENSION_APIS', {}) - - def get_handler(name): return _GLOBAL.get('HANDLERS', {}).get(name, {}) diff --git a/src/spaceone/core/config/default_conf.py b/src/spaceone/core/config/default_conf.py index 92c338f..350d1f8 100644 --- a/src/spaceone/core/config/default_conf.py +++ b/src/spaceone/core/config/default_conf.py @@ -1,9 +1,3 @@ -# Service Description -TITLE = '' -DESCRIPTION = '' -CONTACT = { -} - # Service Configuration PACKAGE = None SERVICE = None @@ -18,20 +12,24 @@ # Unit Test Configuration MOCK_MODE = False +# gRPC Configuration # gRPC Extension APIs -EXTENSION_APIS = { +GRPC_EXTENSION_APIS = { 'spaceone.core.extension.grpc_health': ['GRPCHealth'], 'spaceone.core.extension.server_info': ['ServerInfo'] } -# Uvicorn Options for Rest Server -UVICORN_OPTIONS = { - 'factory': True +# REST Configuration +# REST Application Options +REST_TITLE = '' +REST_DESCRIPTION = '' +REST_CONTACT = { } -# Rest Middlewares +# REST Middlewares REST_MIDDLEWARES = [] +# REST Extension Routers REST_EXTENSION_ROUTERS = [ { 'router_path': 'spaceone.core.fastapi.extension.health:router', @@ -39,6 +37,15 @@ } ] +# REST Uvicorn Options +UVICORN_OPTIONS = { + 'factory': True +} + +# # REST External Swagger +# EXTERNAL_SWAGGER = False +# EXTERNAL_SWAGGER_PATH = [] + # Handler Configuration HANDLERS = { 'authentication': [], diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index e02faad..bb1f348 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -8,7 +8,6 @@ from spaceone.core.logger import set_logger from spaceone.core.opentelemetry import set_tracer, set_metric from spaceone.core.extension.server_info import ServerInfoManager -from spaceone.core.fastapi.openapi import add_external_swagger _LOGGER = logging.getLogger(__name__) @@ -124,10 +123,10 @@ def _init_fast_api(): server_info = ServerInfoManager() return FastAPI( - title=global_conf.get('TITLE', 'Document'), + title=global_conf.get('REST_TITLE', 'Document'), version=server_info.get_version(), - contact=global_conf.get('CONTACT', {}), - description=global_conf.get('DESCRIPTION', ''), + contact=global_conf.get('REST_CONTACT', {}), + description=global_conf.get('REST_DESCRIPTION', ''), ) @@ -135,7 +134,6 @@ def fast_api_app(): app = _init_fast_api() app = _add_middlewares(app) app = _include_routers(app) - app = add_external_swagger(app) return app diff --git a/src/spaceone/core/pygrpc/server.py b/src/spaceone/core/pygrpc/server.py index d2d0408..2f7dcfb 100644 --- a/src/spaceone/core/pygrpc/server.py +++ b/src/spaceone/core/pygrpc/server.py @@ -51,7 +51,7 @@ def _init_services(server): server, service_names = _add_services(server, service_names, proto_conf) # Set Extension Services - proto_conf = config.get_extension_apis() + proto_conf = config.get_global('GRPC_EXTENSION_APIS', {}) server, service_names = _add_services(server, service_names, proto_conf) return server, service_names From be92b5e133f47b84c649de1f0c82ea391634eba4 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Wed, 6 Sep 2023 14:11:26 +0900 Subject: [PATCH 07/15] feat: remove unused config attribute Signed-off-by: ImMin5 (cherry picked from commit 8ad715f5ed935663deeb25e098110e0d7d5d2655) --- src/spaceone/core/config/default_conf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/spaceone/core/config/default_conf.py b/src/spaceone/core/config/default_conf.py index 350d1f8..368b38c 100644 --- a/src/spaceone/core/config/default_conf.py +++ b/src/spaceone/core/config/default_conf.py @@ -42,9 +42,6 @@ 'factory': True } -# # REST External Swagger -# EXTERNAL_SWAGGER = False -# EXTERNAL_SWAGGER_PATH = [] # Handler Configuration HANDLERS = { From 0ef371089619f2c27175c76eb9d9fad94a91cffa Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 22 Sep 2023 09:00:43 +0900 Subject: [PATCH 08/15] feat: move set_pytho_path function from command.py Signed-off-by: ImMin5 (cherry picked from commit c8a58a0d4cf1130ee39c68474ffdc3a097191159) --- src/spaceone/core/utils.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/spaceone/core/utils.py b/src/spaceone/core/utils.py index ffc0081..aef9a31 100644 --- a/src/spaceone/core/utils.py +++ b/src/spaceone/core/utils.py @@ -1,14 +1,18 @@ +import os import re import time import random import string import secrets +import sys import datetime -from urllib.parse import urlparse import yaml import json import hashlib import urllib +import pkg_resources + +from urllib.parse import urlparse from dateutil.parser import isoparse from typing import Tuple from pathlib import Path @@ -494,5 +498,26 @@ def change_dict_with_dot_notation(dict_value: dict, key='', dots=None) -> dict: return dots +def set_python_path(package, module_path): + current_path = os.getcwd() + + if current_path not in sys.path: + sys.path.insert(0, current_path) + + if isinstance(module_path, tuple): + for path in module_path: + if path not in sys.path: + sys.path.insert(0, path) + + if '.' in package: + pkg_resources.declare_namespace(package) + + try: + __import__(package) + except Exception: + raise Exception(f'The package({package}) can not imported. ' + 'Please check the module path.') + + if __name__ == '__main__': pass From 11b7075cb0db9778e0e52c1660c863311fb2d043 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 22 Sep 2023 09:02:38 +0900 Subject: [PATCH 09/15] feat: add set_environment step when start rest server Signed-off-by: ImMin5 (cherry picked from commit e7d2442ff1e2d97bf5229ec2e1dff0bd88e62b75) --- src/spaceone/core/command.py | 42 +++++++++++------------------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/src/spaceone/core/command.py b/src/spaceone/core/command.py index 4ac08a7..e0e7242 100644 --- a/src/spaceone/core/command.py +++ b/src/spaceone/core/command.py @@ -1,11 +1,9 @@ import os import shutil -import sys import unittest from typing import List import click -import pkg_resources from spaceone.core import config, pygrpc, fastapi, utils from spaceone.core import scheduler as scheduler_v1 @@ -48,6 +46,7 @@ def grpc(package, port=None, config_file=None, module_path=None): @click.option('-m', '--module-path', type=click.Path(exists=True), multiple=True, help='Module path') def rest(package, host=None, port=None, config_file=None, module_path=None): """Run a FastAPI REST server""" + _set_environment(package, host, port, config_file, module_path) _set_server_config(package, module_path, port, config_file=config_file) fastapi.serve() @@ -107,35 +106,9 @@ def test(config_file=None, dir=None, failfast=False, scenario: str = None, param RichTestRunner(verbosity=verbose, failfast=failfast).run(full_suite) -def _set_file_config(conf_file): - if conf_file: - config.set_file_conf(conf_file) - - -def _set_python_path(package, module_path): - current_path = os.getcwd() - - if current_path not in sys.path: - sys.path.insert(0, current_path) - - if isinstance(module_path, tuple): - for path in module_path: - if path not in sys.path: - sys.path.insert(0, path) - - if '.' in package: - pkg_resources.declare_namespace(package) - - try: - __import__(package) - except Exception: - raise Exception(f'The package({package}) can not imported. ' - 'Please check the module path.') - - def _set_server_config(package, module_path=None, port=None, config_file=None): # 1. Set a python path - _set_python_path(package, module_path) + utils.set_python_path(package, module_path) # 2. Initialize config from command argument config.init_conf( @@ -147,7 +120,8 @@ def _set_server_config(package, module_path=None, port=None, config_file=None): config.set_service_config() # 4. Merge file conf - _set_file_config(config_file) + if config_file: + config.set_file_conf(config_file) def init_project_file(path, text): @@ -191,5 +165,13 @@ def _print_config(output): print(utils.dump_yaml(data)) +def _set_environment(package, host, port, config_file, module_path): + os.environ['SPACEONE_PACKAGE'] = package + os.environ['SPACEONE_HOST'] = host + os.environ['SPACEONE_MODULE_PATH'] = ', '.join(module_path) + os.environ['SPACEONE_PORT'] = str(port) + os.environ['SPACEONE_CONFIG_FILE'] = config_file + + if __name__ == '__main__': cli() From 15609ca05f7d64b14b067077cc138d23ccb802d7 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 22 Sep 2023 09:03:42 +0900 Subject: [PATCH 10/15] feat: load global conf using env when init_fastapi_app Signed-off-by: ImMin5 (cherry picked from commit e79374e1bafd21c3c9f16bfe8aeee1d09327d8f9) --- src/spaceone/core/fastapi/server.py | 43 +++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index bb1f348..1cc4242 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -1,10 +1,11 @@ import logging -import uvicorn +import os +import uvicorn from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from spaceone.core import config +from spaceone.core import config, utils from spaceone.core.logger import set_logger from spaceone.core.opentelemetry import set_tracer, set_metric from spaceone.core.extension.server_info import ServerInfoManager @@ -130,7 +131,39 @@ def _init_fast_api(): ) +def set_server_config_from_env(): + package = os.environ.get('SPACEONE_PACKAGE') + host = os.environ['SPACEONE_HOST'] + module_path = tuple(os.environ.get('SPACEONE_MODULE_PATH').split(', ')) + port = os.environ.get('SPACEON_PORT') + config_file = os.environ.get('SPACEONE_CONFIG_FILE') + + # 1. Set a python path + utils.set_python_path(package, module_path) + + # 2. Initialize config from command argument + config.init_conf( + package=package, + port=port + ) + + # 3. Get service config from global_conf.py + config.set_service_config() + + # 4. Merge file conf + if config_file: + config.set_file_conf(config_file) + + def fast_api_app(): + set_server_config_from_env() + + # Enable logging configuration\ + set_logger() + + # Set OTel Tracer and Metric| + set_tracer() + app = _init_fast_api() app = _add_middlewares(app) app = _include_routers(app) @@ -140,12 +173,6 @@ def fast_api_app(): def serve(): conf = config.get_global() - # Enable logging configuration\ - set_logger() - - # Set OTel Tracer and Metric| - set_tracer() - uvicorn_options = conf.get('UVICORN_OPTIONS', {}) _LOGGER.info(f'Start REST Server ({config.get_service()}): ' From 58f4d8061fdbe7a9497afa1da7e15057feb0dc1b Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Tue, 17 Oct 2023 15:39:30 +0900 Subject: [PATCH 11/15] feat: add REST_MAX_THREAD_POOL at default_conf Signed-off-by: ImMin5 (cherry picked from commit bb2fc2a40b9d9077ab64317a93c7ad6c7da0c0ac) --- src/spaceone/core/config/default_conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spaceone/core/config/default_conf.py b/src/spaceone/core/config/default_conf.py index 368b38c..7487808 100644 --- a/src/spaceone/core/config/default_conf.py +++ b/src/spaceone/core/config/default_conf.py @@ -23,8 +23,8 @@ # REST Application Options REST_TITLE = '' REST_DESCRIPTION = '' -REST_CONTACT = { -} +REST_CONTACT = {} +REST_MAX_THREAD_POOL = 40 # REST Middlewares REST_MIDDLEWARES = [] From bbb2f33007566d6f47dd8b867fe3a4e04e1a544e Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Tue, 17 Oct 2023 15:40:32 +0900 Subject: [PATCH 12/15] feat: set log and tracer before start server Signed-off-by: ImMin5 (cherry picked from commit 2f88d04422bb4f8674addecef324e9a1093b67c2) --- src/spaceone/core/fastapi/server.py | 42 +++++++---------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index 1cc4242..778695f 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -1,7 +1,7 @@ import logging -import os import uvicorn +from anyio import to_thread from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -122,6 +122,8 @@ def _add_middlewares(app): def _init_fast_api(): global_conf = config.get_global() server_info = ServerInfoManager() + # set thread pool size + to_thread.current_default_thread_limiter().total_tokens = global_conf.get('REST_MAX_THREAD_POOL', 40) return FastAPI( title=global_conf.get('REST_TITLE', 'Document'), @@ -131,39 +133,7 @@ def _init_fast_api(): ) -def set_server_config_from_env(): - package = os.environ.get('SPACEONE_PACKAGE') - host = os.environ['SPACEONE_HOST'] - module_path = tuple(os.environ.get('SPACEONE_MODULE_PATH').split(', ')) - port = os.environ.get('SPACEON_PORT') - config_file = os.environ.get('SPACEONE_CONFIG_FILE') - - # 1. Set a python path - utils.set_python_path(package, module_path) - - # 2. Initialize config from command argument - config.init_conf( - package=package, - port=port - ) - - # 3. Get service config from global_conf.py - config.set_service_config() - - # 4. Merge file conf - if config_file: - config.set_file_conf(config_file) - - def fast_api_app(): - set_server_config_from_env() - - # Enable logging configuration\ - set_logger() - - # Set OTel Tracer and Metric| - set_tracer() - app = _init_fast_api() app = _add_middlewares(app) app = _include_routers(app) @@ -173,6 +143,12 @@ def fast_api_app(): def serve(): conf = config.get_global() + # Enable logging configuration\ + set_logger() + + # Set OTel Tracer and Metric| + set_tracer() + uvicorn_options = conf.get('UVICORN_OPTIONS', {}) _LOGGER.info(f'Start REST Server ({config.get_service()}): ' From 225b670f6951546525960b29e30b7aa421a789bc Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Tue, 17 Oct 2023 15:42:35 +0900 Subject: [PATCH 13/15] feat: remove unused function Signed-off-by: ImMin5 (cherry picked from commit fcb1d3cfa81c62eff9c700793d56ec7927ebe379) --- src/spaceone/core/command.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/spaceone/core/command.py b/src/spaceone/core/command.py index e0e7242..2be1f19 100644 --- a/src/spaceone/core/command.py +++ b/src/spaceone/core/command.py @@ -46,7 +46,6 @@ def grpc(package, port=None, config_file=None, module_path=None): @click.option('-m', '--module-path', type=click.Path(exists=True), multiple=True, help='Module path') def rest(package, host=None, port=None, config_file=None, module_path=None): """Run a FastAPI REST server""" - _set_environment(package, host, port, config_file, module_path) _set_server_config(package, module_path, port, config_file=config_file) fastapi.serve() @@ -165,13 +164,5 @@ def _print_config(output): print(utils.dump_yaml(data)) -def _set_environment(package, host, port, config_file, module_path): - os.environ['SPACEONE_PACKAGE'] = package - os.environ['SPACEONE_HOST'] = host - os.environ['SPACEONE_MODULE_PATH'] = ', '.join(module_path) - os.environ['SPACEONE_PORT'] = str(port) - os.environ['SPACEONE_CONFIG_FILE'] = config_file - - if __name__ == '__main__': cli() From 058e1b3e92ccb1dd1b27eb689c963114917bee6b Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Thu, 19 Oct 2023 11:30:18 +0900 Subject: [PATCH 14/15] feat: modify _set_python_path location to command.py from utils.py Signed-off-by: ImMin5 (cherry picked from commit 4e07f7388d9b87be19e484e9d9148ce6a4554590) --- src/spaceone/core/command.py | 25 +++++++++++++++++++++++- src/spaceone/core/config/default_conf.py | 1 - src/spaceone/core/fastapi/server.py | 3 --- src/spaceone/core/utils.py | 21 -------------------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/spaceone/core/command.py b/src/spaceone/core/command.py index 2be1f19..6f7311f 100644 --- a/src/spaceone/core/command.py +++ b/src/spaceone/core/command.py @@ -1,9 +1,11 @@ import os import shutil +import sys import unittest from typing import List import click +import pkg_resources from spaceone.core import config, pygrpc, fastapi, utils from spaceone.core import scheduler as scheduler_v1 @@ -105,9 +107,30 @@ def test(config_file=None, dir=None, failfast=False, scenario: str = None, param RichTestRunner(verbosity=verbose, failfast=failfast).run(full_suite) +def _set_python_path(package, module_path): + current_path = os.getcwd() + + if current_path not in sys.path: + sys.path.insert(0, current_path) + + if isinstance(module_path, tuple): + for path in module_path: + if path not in sys.path: + sys.path.insert(0, path) + + if '.' in package: + pkg_resources.declare_namespace(package) + + try: + __import__(package) + except Exception: + raise Exception(f'The package({package}) can not imported. ' + 'Please check the module path.') + + def _set_server_config(package, module_path=None, port=None, config_file=None): # 1. Set a python path - utils.set_python_path(package, module_path) + _set_python_path(package, module_path) # 2. Initialize config from command argument config.init_conf( diff --git a/src/spaceone/core/config/default_conf.py b/src/spaceone/core/config/default_conf.py index 7487808..09816ad 100644 --- a/src/spaceone/core/config/default_conf.py +++ b/src/spaceone/core/config/default_conf.py @@ -24,7 +24,6 @@ REST_TITLE = '' REST_DESCRIPTION = '' REST_CONTACT = {} -REST_MAX_THREAD_POOL = 40 # REST Middlewares REST_MIDDLEWARES = [] diff --git a/src/spaceone/core/fastapi/server.py b/src/spaceone/core/fastapi/server.py index 778695f..d41c203 100644 --- a/src/spaceone/core/fastapi/server.py +++ b/src/spaceone/core/fastapi/server.py @@ -1,7 +1,6 @@ import logging import uvicorn -from anyio import to_thread from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -122,8 +121,6 @@ def _add_middlewares(app): def _init_fast_api(): global_conf = config.get_global() server_info = ServerInfoManager() - # set thread pool size - to_thread.current_default_thread_limiter().total_tokens = global_conf.get('REST_MAX_THREAD_POOL', 40) return FastAPI( title=global_conf.get('REST_TITLE', 'Document'), diff --git a/src/spaceone/core/utils.py b/src/spaceone/core/utils.py index aef9a31..b5e8bd9 100644 --- a/src/spaceone/core/utils.py +++ b/src/spaceone/core/utils.py @@ -498,26 +498,5 @@ def change_dict_with_dot_notation(dict_value: dict, key='', dots=None) -> dict: return dots -def set_python_path(package, module_path): - current_path = os.getcwd() - - if current_path not in sys.path: - sys.path.insert(0, current_path) - - if isinstance(module_path, tuple): - for path in module_path: - if path not in sys.path: - sys.path.insert(0, path) - - if '.' in package: - pkg_resources.declare_namespace(package) - - try: - __import__(package) - except Exception: - raise Exception(f'The package({package}) can not imported. ' - 'Please check the module path.') - - if __name__ == '__main__': pass From 8ddce12a88136d048b9198fdaeadf873cb4f1f97 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Thu, 19 Oct 2023 15:06:38 +0900 Subject: [PATCH 15/15] feat: add token parameter at _get_connection_metadata method Signed-off-by: ImMin5 (cherry picked from commit fbf5e2b7af62b3d5e2150737e626fc2120a5612b) --- src/spaceone/core/connector/space_connector.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/spaceone/core/connector/space_connector.py b/src/spaceone/core/connector/space_connector.py index a44b14f..8015bba 100644 --- a/src/spaceone/core/connector/space_connector.py +++ b/src/spaceone/core/connector/space_connector.py @@ -39,13 +39,15 @@ def dispatch(self, method: str, params: dict = None, **kwargs): return self._call_api(method, params, **kwargs) def _call_api(self, method: str, params: dict = None, **kwargs): + token = kwargs.get('token') + self._check_mock_mode(method) resource, verb = self._parse_method(method) self._check_method(resource, verb) params = params or {} - kwargs['metadata'] = self._get_connection_metadata() + kwargs['metadata'] = self._get_connection_metadata(token) response_or_iterator = getattr(getattr(self._client, resource), verb)(params, **kwargs) @@ -89,11 +91,13 @@ def _generate_response(self, response_iterator): for response in response_iterator: yield self._change_message(response) - def _get_connection_metadata(self): + def _get_connection_metadata(self, token=None): metadata = [] - if self._token: - metadata.append(('token', self._token)) + if token: + metadata.append(('token', token)) + elif self._token: + metadata.append(('token', self._token)) elif token := self.transaction.meta.get('token'): metadata.append(('token', token))