From a55d48a1a56d39c06a85fb2077e4cd1d673582f4 Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Mon, 18 Nov 2019 17:08:40 +0200 Subject: [PATCH 1/6] split perf tests --- ..._tests.py => build_perf_hello_world_js.py} | 4 +- .../perf/tooling/build_perf_hello_world_ng.py | 159 ++++++++++++++++++ .../tooling/build_perf_master_detail_ng.py | 159 ++++++++++++++++++ 3 files changed, 319 insertions(+), 3 deletions(-) rename tests/perf/tooling/{build_perf_tests.py => build_perf_hello_world_js.py} (97%) create mode 100644 tests/perf/tooling/build_perf_hello_world_ng.py create mode 100644 tests/perf/tooling/build_perf_master_detail_ng.py diff --git a/tests/perf/tooling/build_perf_tests.py b/tests/perf/tooling/build_perf_hello_world_js.py similarity index 97% rename from tests/perf/tooling/build_perf_tests.py rename to tests/perf/tooling/build_perf_hello_world_js.py index 7fae5281..a77b0720 100644 --- a/tests/perf/tooling/build_perf_tests.py +++ b/tests/perf/tooling/build_perf_hello_world_js.py @@ -30,9 +30,7 @@ # noinspection PyMethodMayBeStatic,PyUnusedLocal class PrepareAndBuildPerfTests(TnsTest): TEST_DATA = [ - ('hello-world-js', Template.HELLO_WORLD_JS.local_package, Changes.JSHelloWord.JS), - ('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS), - ('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS), + ('hello-world-js', Template.HELLO_WORLD_JS.local_package, Changes.JSHelloWord.JS) ] @classmethod diff --git a/tests/perf/tooling/build_perf_hello_world_ng.py b/tests/perf/tooling/build_perf_hello_world_ng.py new file mode 100644 index 00000000..880b2569 --- /dev/null +++ b/tests/perf/tooling/build_perf_hello_world_ng.py @@ -0,0 +1,159 @@ +# pylint: disable=unused-argument +# pylint: disable=undefined-variable + +import json +import os +import unittest + +from parameterized import parameterized + +from core.base_test.tns_test import TnsTest +from core.enums.os_type import OSType +from core.enums.platform_type import Platform +from core.settings import Settings +from core.utils.file_utils import Folder, File +from core.utils.gradle import Gradle +from core.utils.json_utils import JsonUtils +from core.utils.npm import Npm +from core.utils.perf_utils import PerfUtils +from core.utils.xcode import Xcode +from data.changes import Changes, Sync +from data.templates import Template +from products.nativescript.tns import Tns + +RETRY_COUNT = 3 +TOLERANCE = 0.20 +APP_NAME = Settings.AppName.DEFAULT +EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) + + +# noinspection PyMethodMayBeStatic,PyUnusedLocal +class PrepareAndBuildPerfTests(TnsTest): + TEST_DATA = [ + ('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS) + ] + + @classmethod + def setUpClass(cls): + TnsTest.setUpClass() + + def setUp(self): + TnsTest.setUp(self) + + @classmethod + def tearDownClass(cls): + TnsTest.tearDownClass() + + @parameterized.expand(TEST_DATA) + def test_001_prepare_data(self, template, template_package, change_set): + android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) + ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) + Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, + change_set=change_set, result_file=android_result_file) + Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, + change_set=change_set, result_file=ios_result_file) + + @parameterized.expand(TEST_DATA) + def test_200_prepare_android_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_201_prepare_ios_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + + @parameterized.expand(TEST_DATA) + def test_300_build_android_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_301_build_ios_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + + @parameterized.expand(TEST_DATA) + def test_310_build_android_incremental(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_311_build_ios_incremental(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' + + +class PrepareBuildInfo(object): + prepare_initial = 0 + prepare_skip = 0 + prepare_incremental = 0 + build_initial = 0 + build_incremental = 0 + + +class Helpers(object): + @staticmethod + def prepare_and_build(template, platform, change_set, result_file): + prepare_initial = 0 + build_initial = 0 + build_incremental = 0 + for _ in range(RETRY_COUNT): + Tns.kill() + Gradle.kill() + Npm.cache_clean() + Xcode.cache_clean() + Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) + Tns.create(app_name=APP_NAME, template=template, update=True) + if platform == Platform.ANDROID: + Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) + elif platform == Platform.IOS: + Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) + else: + raise Exception('Unknown platform: ' + str(platform)) + + # Prepare + time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration + prepare_initial = prepare_initial + time + + # Build + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_initial = build_initial + time + Sync.replace(app_name=APP_NAME, change_set=change_set) + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_incremental = build_incremental + time + + # Calculate averages + result = PrepareBuildInfo() + result.prepare_initial = prepare_initial / RETRY_COUNT + result.build_initial = build_initial / RETRY_COUNT + result.build_incremental = build_incremental / RETRY_COUNT + + # Save to results file + File.delete(path=result_file) + result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) + File.write(path=result_file, text=str(result_json)) + + @staticmethod + def get_result_file_name(template, platform): + result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) + return result_file + + @staticmethod + def get_actual_result(template, platform, entry): + result_file = Helpers.get_result_file_name(template, platform) + return JsonUtils.read(result_file)[entry] + + @staticmethod + def get_expected_result(template, platform, entry): + platform = str(platform) + return EXPECTED_RESULTS[template][platform][entry] diff --git a/tests/perf/tooling/build_perf_master_detail_ng.py b/tests/perf/tooling/build_perf_master_detail_ng.py new file mode 100644 index 00000000..7b512964 --- /dev/null +++ b/tests/perf/tooling/build_perf_master_detail_ng.py @@ -0,0 +1,159 @@ +# pylint: disable=unused-argument +# pylint: disable=undefined-variable + +import json +import os +import unittest + +from parameterized import parameterized + +from core.base_test.tns_test import TnsTest +from core.enums.os_type import OSType +from core.enums.platform_type import Platform +from core.settings import Settings +from core.utils.file_utils import Folder, File +from core.utils.gradle import Gradle +from core.utils.json_utils import JsonUtils +from core.utils.npm import Npm +from core.utils.perf_utils import PerfUtils +from core.utils.xcode import Xcode +from data.changes import Changes, Sync +from data.templates import Template +from products.nativescript.tns import Tns + +RETRY_COUNT = 3 +TOLERANCE = 0.20 +APP_NAME = Settings.AppName.DEFAULT +EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) + + +# noinspection PyMethodMayBeStatic,PyUnusedLocal +class PrepareAndBuildPerfTests(TnsTest): + TEST_DATA = [ + ('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS) + ] + + @classmethod + def setUpClass(cls): + TnsTest.setUpClass() + + def setUp(self): + TnsTest.setUp(self) + + @classmethod + def tearDownClass(cls): + TnsTest.tearDownClass() + + @parameterized.expand(TEST_DATA) + def test_001_prepare_data(self, template, template_package, change_set): + android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) + ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) + Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, + change_set=change_set, result_file=android_result_file) + Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, + change_set=change_set, result_file=ios_result_file) + + @parameterized.expand(TEST_DATA) + def test_200_prepare_android_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_201_prepare_ios_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + + @parameterized.expand(TEST_DATA) + def test_300_build_android_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_301_build_ios_initial(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + + @parameterized.expand(TEST_DATA) + def test_310_build_android_incremental(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_311_build_ios_incremental(self, template, template_package, change_set): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' + + +class PrepareBuildInfo(object): + prepare_initial = 0 + prepare_skip = 0 + prepare_incremental = 0 + build_initial = 0 + build_incremental = 0 + + +class Helpers(object): + @staticmethod + def prepare_and_build(template, platform, change_set, result_file): + prepare_initial = 0 + build_initial = 0 + build_incremental = 0 + for _ in range(RETRY_COUNT): + Tns.kill() + Gradle.kill() + Npm.cache_clean() + Xcode.cache_clean() + Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) + Tns.create(app_name=APP_NAME, template=template, update=True) + if platform == Platform.ANDROID: + Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) + elif platform == Platform.IOS: + Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) + else: + raise Exception('Unknown platform: ' + str(platform)) + + # Prepare + time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration + prepare_initial = prepare_initial + time + + # Build + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_initial = build_initial + time + Sync.replace(app_name=APP_NAME, change_set=change_set) + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_incremental = build_incremental + time + + # Calculate averages + result = PrepareBuildInfo() + result.prepare_initial = prepare_initial / RETRY_COUNT + result.build_initial = build_initial / RETRY_COUNT + result.build_incremental = build_incremental / RETRY_COUNT + + # Save to results file + File.delete(path=result_file) + result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) + File.write(path=result_file, text=str(result_json)) + + @staticmethod + def get_result_file_name(template, platform): + result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) + return result_file + + @staticmethod + def get_actual_result(template, platform, entry): + result_file = Helpers.get_result_file_name(template, platform) + return JsonUtils.read(result_file)[entry] + + @staticmethod + def get_expected_result(template, platform, entry): + platform = str(platform) + return EXPECTED_RESULTS[template][platform][entry] From 12b9cb458b3aa4494ec2bd2eb1de505be82c9d77 Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Tue, 19 Nov 2019 10:19:34 +0200 Subject: [PATCH 2/6] perf split --- products/nativescript/perf_helpers.py | 81 +++++++++++++++++++ .../perf/tooling/build_perf_hello_world_js.py | 80 +----------------- .../perf/tooling/build_perf_hello_world_ng.py | 80 +----------------- .../tooling/build_perf_master_detail_ng.py | 70 +--------------- 4 files changed, 86 insertions(+), 225 deletions(-) create mode 100644 products/nativescript/perf_helpers.py diff --git a/products/nativescript/perf_helpers.py b/products/nativescript/perf_helpers.py new file mode 100644 index 00000000..bd059b7d --- /dev/null +++ b/products/nativescript/perf_helpers.py @@ -0,0 +1,81 @@ +import json +import os +from core.enums.platform_type import Platform +from core.settings import Settings +from core.utils.file_utils import Folder, File +from core.utils.gradle import Gradle +from core.utils.json_utils import JsonUtils +from core.utils.npm import Npm +from core.utils.xcode import Xcode +from data.changes import Sync +from products.nativescript.tns import Tns + +RETRY_COUNT = 3 +# TOLERANCE = 0.20 +APP_NAME = Settings.AppName.DEFAULT +EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) + +class PrepareBuildInfo(object): + prepare_initial = 0 + prepare_skip = 0 + prepare_incremental = 0 + build_initial = 0 + build_incremental = 0 + + +class Helpers(object): + @staticmethod + def prepare_and_build(template, platform, change_set, result_file): + prepare_initial = 0 + build_initial = 0 + build_incremental = 0 + for _ in range(RETRY_COUNT): + Tns.kill() + Gradle.kill() + Npm.cache_clean() + Xcode.cache_clean() + Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) + Tns.create(app_name=APP_NAME, template=template, update=True) + if platform == Platform.ANDROID: + Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) + elif platform == Platform.IOS: + Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) + else: + raise Exception('Unknown platform: ' + str(platform)) + + # Prepare + time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration + prepare_initial = prepare_initial + time + + # Build + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_initial = build_initial + time + Sync.replace(app_name=APP_NAME, change_set=change_set) + time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration + build_incremental = build_incremental + time + + # Calculate averages + result = PrepareBuildInfo() + result.prepare_initial = prepare_initial / RETRY_COUNT + result.build_initial = build_initial / RETRY_COUNT + result.build_incremental = build_incremental / RETRY_COUNT + + # Save to results file + File.delete(path=result_file) + result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) + File.write(path=result_file, text=str(result_json)) + + @staticmethod + def get_result_file_name(template, platform): + result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) + return result_file + + @staticmethod + def get_actual_result(template, platform, entry): + result_file = Helpers.get_result_file_name(template, platform) + return JsonUtils.read(result_file)[entry] + + @staticmethod + def get_expected_result(template, platform, entry): + platform = str(platform) + return EXPECTED_RESULTS[template][platform][entry] \ No newline at end of file diff --git a/tests/perf/tooling/build_perf_hello_world_js.py b/tests/perf/tooling/build_perf_hello_world_js.py index a77b0720..4307aa1f 100644 --- a/tests/perf/tooling/build_perf_hello_world_js.py +++ b/tests/perf/tooling/build_perf_hello_world_js.py @@ -1,8 +1,6 @@ # pylint: disable=unused-argument # pylint: disable=undefined-variable -import json -import os import unittest from parameterized import parameterized @@ -11,20 +9,12 @@ from core.enums.os_type import OSType from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.file_utils import Folder, File -from core.utils.gradle import Gradle -from core.utils.json_utils import JsonUtils -from core.utils.npm import Npm from core.utils.perf_utils import PerfUtils -from core.utils.xcode import Xcode -from data.changes import Changes, Sync +from data.changes import Changes from data.templates import Template -from products.nativescript.tns import Tns +from products.nativescript.perf_helpers import Helpers -RETRY_COUNT = 3 TOLERANCE = 0.20 -APP_NAME = Settings.AppName.DEFAULT -EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) # noinspection PyMethodMayBeStatic,PyUnusedLocal @@ -91,69 +81,3 @@ def test_311_build_ios_incremental(self, template, template_package, change_set) actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' - - -class PrepareBuildInfo(object): - prepare_initial = 0 - prepare_skip = 0 - prepare_incremental = 0 - build_initial = 0 - build_incremental = 0 - - -class Helpers(object): - @staticmethod - def prepare_and_build(template, platform, change_set, result_file): - prepare_initial = 0 - build_initial = 0 - build_incremental = 0 - for _ in range(RETRY_COUNT): - Tns.kill() - Gradle.kill() - Npm.cache_clean() - Xcode.cache_clean() - Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) - Tns.create(app_name=APP_NAME, template=template, update=True) - if platform == Platform.ANDROID: - Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) - elif platform == Platform.IOS: - Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) - else: - raise Exception('Unknown platform: ' + str(platform)) - - # Prepare - time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration - prepare_initial = prepare_initial + time - - # Build - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_initial = build_initial + time - Sync.replace(app_name=APP_NAME, change_set=change_set) - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_incremental = build_incremental + time - - # Calculate averages - result = PrepareBuildInfo() - result.prepare_initial = prepare_initial / RETRY_COUNT - result.build_initial = build_initial / RETRY_COUNT - result.build_incremental = build_incremental / RETRY_COUNT - - # Save to results file - File.delete(path=result_file) - result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) - File.write(path=result_file, text=str(result_json)) - - @staticmethod - def get_result_file_name(template, platform): - result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) - return result_file - - @staticmethod - def get_actual_result(template, platform, entry): - result_file = Helpers.get_result_file_name(template, platform) - return JsonUtils.read(result_file)[entry] - - @staticmethod - def get_expected_result(template, platform, entry): - platform = str(platform) - return EXPECTED_RESULTS[template][platform][entry] diff --git a/tests/perf/tooling/build_perf_hello_world_ng.py b/tests/perf/tooling/build_perf_hello_world_ng.py index 880b2569..23a82d95 100644 --- a/tests/perf/tooling/build_perf_hello_world_ng.py +++ b/tests/perf/tooling/build_perf_hello_world_ng.py @@ -1,8 +1,6 @@ # pylint: disable=unused-argument # pylint: disable=undefined-variable -import json -import os import unittest from parameterized import parameterized @@ -11,20 +9,12 @@ from core.enums.os_type import OSType from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.file_utils import Folder, File -from core.utils.gradle import Gradle -from core.utils.json_utils import JsonUtils -from core.utils.npm import Npm from core.utils.perf_utils import PerfUtils -from core.utils.xcode import Xcode -from data.changes import Changes, Sync +from data.changes import Changes from data.templates import Template -from products.nativescript.tns import Tns +from products.nativescript.perf_helpers import Helpers -RETRY_COUNT = 3 TOLERANCE = 0.20 -APP_NAME = Settings.AppName.DEFAULT -EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) # noinspection PyMethodMayBeStatic,PyUnusedLocal @@ -91,69 +81,3 @@ def test_311_build_ios_incremental(self, template, template_package, change_set) actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' - - -class PrepareBuildInfo(object): - prepare_initial = 0 - prepare_skip = 0 - prepare_incremental = 0 - build_initial = 0 - build_incremental = 0 - - -class Helpers(object): - @staticmethod - def prepare_and_build(template, platform, change_set, result_file): - prepare_initial = 0 - build_initial = 0 - build_incremental = 0 - for _ in range(RETRY_COUNT): - Tns.kill() - Gradle.kill() - Npm.cache_clean() - Xcode.cache_clean() - Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) - Tns.create(app_name=APP_NAME, template=template, update=True) - if platform == Platform.ANDROID: - Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) - elif platform == Platform.IOS: - Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) - else: - raise Exception('Unknown platform: ' + str(platform)) - - # Prepare - time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration - prepare_initial = prepare_initial + time - - # Build - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_initial = build_initial + time - Sync.replace(app_name=APP_NAME, change_set=change_set) - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_incremental = build_incremental + time - - # Calculate averages - result = PrepareBuildInfo() - result.prepare_initial = prepare_initial / RETRY_COUNT - result.build_initial = build_initial / RETRY_COUNT - result.build_incremental = build_incremental / RETRY_COUNT - - # Save to results file - File.delete(path=result_file) - result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) - File.write(path=result_file, text=str(result_json)) - - @staticmethod - def get_result_file_name(template, platform): - result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) - return result_file - - @staticmethod - def get_actual_result(template, platform, entry): - result_file = Helpers.get_result_file_name(template, platform) - return JsonUtils.read(result_file)[entry] - - @staticmethod - def get_expected_result(template, platform, entry): - platform = str(platform) - return EXPECTED_RESULTS[template][platform][entry] diff --git a/tests/perf/tooling/build_perf_master_detail_ng.py b/tests/perf/tooling/build_perf_master_detail_ng.py index 7b512964..a7b24e9e 100644 --- a/tests/perf/tooling/build_perf_master_detail_ng.py +++ b/tests/perf/tooling/build_perf_master_detail_ng.py @@ -19,12 +19,10 @@ from core.utils.xcode import Xcode from data.changes import Changes, Sync from data.templates import Template +from products.nativescript.perf_helpers import Helpers from products.nativescript.tns import Tns -RETRY_COUNT = 3 TOLERANCE = 0.20 -APP_NAME = Settings.AppName.DEFAULT -EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) # noinspection PyMethodMayBeStatic,PyUnusedLocal @@ -91,69 +89,3 @@ def test_311_build_ios_incremental(self, template, template_package, change_set) actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' - - -class PrepareBuildInfo(object): - prepare_initial = 0 - prepare_skip = 0 - prepare_incremental = 0 - build_initial = 0 - build_incremental = 0 - - -class Helpers(object): - @staticmethod - def prepare_and_build(template, platform, change_set, result_file): - prepare_initial = 0 - build_initial = 0 - build_incremental = 0 - for _ in range(RETRY_COUNT): - Tns.kill() - Gradle.kill() - Npm.cache_clean() - Xcode.cache_clean() - Folder.clean(folder=os.path.join(Settings.TEST_RUN_HOME, APP_NAME)) - Tns.create(app_name=APP_NAME, template=template, update=True) - if platform == Platform.ANDROID: - Tns.platform_add_android(app_name=APP_NAME, framework_path=Settings.Android.FRAMEWORK_PATH) - elif platform == Platform.IOS: - Tns.platform_add_ios(app_name=APP_NAME, framework_path=Settings.IOS.FRAMEWORK_PATH) - else: - raise Exception('Unknown platform: ' + str(platform)) - - # Prepare - time = Tns.prepare(app_name=APP_NAME, platform=platform, bundle=True).duration - prepare_initial = prepare_initial + time - - # Build - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_initial = build_initial + time - Sync.replace(app_name=APP_NAME, change_set=change_set) - time = Tns.build(app_name=APP_NAME, platform=platform, bundle=True).duration - build_incremental = build_incremental + time - - # Calculate averages - result = PrepareBuildInfo() - result.prepare_initial = prepare_initial / RETRY_COUNT - result.build_initial = build_initial / RETRY_COUNT - result.build_incremental = build_incremental / RETRY_COUNT - - # Save to results file - File.delete(path=result_file) - result_json = json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) - File.write(path=result_file, text=str(result_json)) - - @staticmethod - def get_result_file_name(template, platform): - result_file = os.path.join(Settings.TEST_OUT_HOME, '{0}_{1}.json'.format(template, str(platform))) - return result_file - - @staticmethod - def get_actual_result(template, platform, entry): - result_file = Helpers.get_result_file_name(template, platform) - return JsonUtils.read(result_file)[entry] - - @staticmethod - def get_expected_result(template, platform, entry): - platform = str(platform) - return EXPECTED_RESULTS[template][platform][entry] From 2791e3a12c66d5a087dfe4c301033a6742fc3a33 Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Tue, 19 Nov 2019 13:45:57 +0200 Subject: [PATCH 3/6] perf split --- products/nativescript/perf_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/nativescript/perf_helpers.py b/products/nativescript/perf_helpers.py index bd059b7d..b897524c 100644 --- a/products/nativescript/perf_helpers.py +++ b/products/nativescript/perf_helpers.py @@ -11,10 +11,10 @@ from products.nativescript.tns import Tns RETRY_COUNT = 3 -# TOLERANCE = 0.20 APP_NAME = Settings.AppName.DEFAULT EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) + class PrepareBuildInfo(object): prepare_initial = 0 prepare_skip = 0 @@ -78,4 +78,4 @@ def get_actual_result(template, platform, entry): @staticmethod def get_expected_result(template, platform, entry): platform = str(platform) - return EXPECTED_RESULTS[template][platform][entry] \ No newline at end of file + return EXPECTED_RESULTS[template][platform][entry] From 98d367b7c962fb52146efea583f0ced010611a79 Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Tue, 19 Nov 2019 14:09:50 +0200 Subject: [PATCH 4/6] pylint error --- tests/perf/tooling/build_perf_master_detail_ng.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/perf/tooling/build_perf_master_detail_ng.py b/tests/perf/tooling/build_perf_master_detail_ng.py index a7b24e9e..9d4ffda1 100644 --- a/tests/perf/tooling/build_perf_master_detail_ng.py +++ b/tests/perf/tooling/build_perf_master_detail_ng.py @@ -1,8 +1,6 @@ # pylint: disable=unused-argument # pylint: disable=undefined-variable -import json -import os import unittest from parameterized import parameterized @@ -11,16 +9,10 @@ from core.enums.os_type import OSType from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.file_utils import Folder, File -from core.utils.gradle import Gradle -from core.utils.json_utils import JsonUtils -from core.utils.npm import Npm from core.utils.perf_utils import PerfUtils -from core.utils.xcode import Xcode -from data.changes import Changes, Sync +from data.changes import Changes from data.templates import Template from products.nativescript.perf_helpers import Helpers -from products.nativescript.tns import Tns TOLERANCE = 0.20 From e8626e71dc4c61f4e6d2ddf86d415970ce7b4ccb Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Wed, 20 Nov 2019 11:21:33 +0200 Subject: [PATCH 5/6] perf tests --- products/nativescript/perf_helpers.py | 47 +++++++++++++++++++ .../perf/tooling/build_perf_hello_world_js.py | 35 +++----------- .../perf/tooling/build_perf_hello_world_ng.py | 35 +++----------- .../tooling/build_perf_master_detail_ng.py | 35 +++----------- 4 files changed, 68 insertions(+), 84 deletions(-) diff --git a/products/nativescript/perf_helpers.py b/products/nativescript/perf_helpers.py index b897524c..54bb0137 100644 --- a/products/nativescript/perf_helpers.py +++ b/products/nativescript/perf_helpers.py @@ -6,11 +6,13 @@ from core.utils.gradle import Gradle from core.utils.json_utils import JsonUtils from core.utils.npm import Npm +from core.utils.perf_utils import PerfUtils from core.utils.xcode import Xcode from data.changes import Sync from products.nativescript.tns import Tns RETRY_COUNT = 3 +TOLERANCE = 0.20 APP_NAME = Settings.AppName.DEFAULT EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) @@ -79,3 +81,48 @@ def get_actual_result(template, platform, entry): def get_expected_result(template, platform, entry): platform = str(platform) return EXPECTED_RESULTS[template][platform][entry] + + @staticmethod + def prepare_data(template, template_package, change_set): + android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) + ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) + Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, + change_set=change_set, result_file=android_result_file) + Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, + change_set=change_set, result_file=ios_result_file) + + @staticmethod + def prepare_android_initial(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + + @staticmethod + def prepare_ios_initial(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + + @staticmethod + def build_android_initial(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + + @staticmethod + def build_ios_initial(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + + @staticmethod + def build_android_incremental(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + + @staticmethod + def build_ios_incremental(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' diff --git a/tests/perf/tooling/build_perf_hello_world_js.py b/tests/perf/tooling/build_perf_hello_world_js.py index 4307aa1f..fb265c1d 100644 --- a/tests/perf/tooling/build_perf_hello_world_js.py +++ b/tests/perf/tooling/build_perf_hello_world_js.py @@ -7,15 +7,11 @@ from core.base_test.tns_test import TnsTest from core.enums.os_type import OSType -from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.perf_utils import PerfUtils from data.changes import Changes from data.templates import Template from products.nativescript.perf_helpers import Helpers -TOLERANCE = 0.20 - # noinspection PyMethodMayBeStatic,PyUnusedLocal class PrepareAndBuildPerfTests(TnsTest): @@ -36,48 +32,31 @@ def tearDownClass(cls): @parameterized.expand(TEST_DATA) def test_001_prepare_data(self, template, template_package, change_set): - android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) - ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) - Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, - change_set=change_set, result_file=android_result_file) - Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, - change_set=change_set, result_file=ios_result_file) + Helpers.prepare_data(template, template_package, change_set) @parameterized.expand(TEST_DATA) def test_200_prepare_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + Helpers.prepare_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_201_prepare_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + Helpers.prepare_ios_initial(template) @parameterized.expand(TEST_DATA) def test_300_build_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + Helpers.build_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_301_build_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + Helpers.build_ios_initial(template) @parameterized.expand(TEST_DATA) def test_310_build_android_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + Helpers.build_android_incremental(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_311_build_ios_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' + Helpers.build_ios_incremental(template) diff --git a/tests/perf/tooling/build_perf_hello_world_ng.py b/tests/perf/tooling/build_perf_hello_world_ng.py index 23a82d95..a5dd78d8 100644 --- a/tests/perf/tooling/build_perf_hello_world_ng.py +++ b/tests/perf/tooling/build_perf_hello_world_ng.py @@ -7,15 +7,11 @@ from core.base_test.tns_test import TnsTest from core.enums.os_type import OSType -from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.perf_utils import PerfUtils from data.changes import Changes from data.templates import Template from products.nativescript.perf_helpers import Helpers -TOLERANCE = 0.20 - # noinspection PyMethodMayBeStatic,PyUnusedLocal class PrepareAndBuildPerfTests(TnsTest): @@ -36,48 +32,31 @@ def tearDownClass(cls): @parameterized.expand(TEST_DATA) def test_001_prepare_data(self, template, template_package, change_set): - android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) - ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) - Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, - change_set=change_set, result_file=android_result_file) - Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, - change_set=change_set, result_file=ios_result_file) + Helpers.prepare_data(template, template_package, change_set) @parameterized.expand(TEST_DATA) def test_200_prepare_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + Helpers.prepare_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_201_prepare_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + Helpers.prepare_ios_initial(template) @parameterized.expand(TEST_DATA) def test_300_build_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + Helpers.build_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_301_build_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + Helpers.build_ios_initial(template) @parameterized.expand(TEST_DATA) def test_310_build_android_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + Helpers.build_android_incremental(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_311_build_ios_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' + Helpers.build_ios_incremental(template) diff --git a/tests/perf/tooling/build_perf_master_detail_ng.py b/tests/perf/tooling/build_perf_master_detail_ng.py index 9d4ffda1..26b2bc53 100644 --- a/tests/perf/tooling/build_perf_master_detail_ng.py +++ b/tests/perf/tooling/build_perf_master_detail_ng.py @@ -7,15 +7,11 @@ from core.base_test.tns_test import TnsTest from core.enums.os_type import OSType -from core.enums.platform_type import Platform from core.settings import Settings -from core.utils.perf_utils import PerfUtils from data.changes import Changes from data.templates import Template from products.nativescript.perf_helpers import Helpers -TOLERANCE = 0.20 - # noinspection PyMethodMayBeStatic,PyUnusedLocal class PrepareAndBuildPerfTests(TnsTest): @@ -36,48 +32,31 @@ def tearDownClass(cls): @parameterized.expand(TEST_DATA) def test_001_prepare_data(self, template, template_package, change_set): - android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) - ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) - Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, - change_set=change_set, result_file=android_result_file) - Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, - change_set=change_set, result_file=ios_result_file) + Helpers.prepare_data(template, template_package, change_set) @parameterized.expand(TEST_DATA) def test_200_prepare_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + Helpers.prepare_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_201_prepare_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + Helpers.prepare_ios_initial(template) @parameterized.expand(TEST_DATA) def test_300_build_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + Helpers.build_android_initial(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_301_build_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + Helpers.build_ios_initial(template) @parameterized.expand(TEST_DATA) def test_310_build_android_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + Helpers.build_android_incremental(template) @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') def test_311_build_ios_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' + Helpers.build_ios_incremental(template) From 4d3039a9bd5c215499d0d9b0574a22b1a2f89727 Mon Sep 17 00:00:00 2001 From: Miroslava Ivanova Date: Wed, 20 Nov 2019 15:28:58 +0200 Subject: [PATCH 6/6] pref --- products/nativescript/perf_helpers.py | 6 +-- .../perf/tooling/build_perf_hello_world_js.py | 42 +++++++------------ 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/products/nativescript/perf_helpers.py b/products/nativescript/perf_helpers.py index 54bb0137..b7479f01 100644 --- a/products/nativescript/perf_helpers.py +++ b/products/nativescript/perf_helpers.py @@ -83,12 +83,12 @@ def get_expected_result(template, platform, entry): return EXPECTED_RESULTS[template][platform][entry] @staticmethod - def prepare_data(template, template_package, change_set): + def prepare_data(template, change_set): android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) - Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, + Helpers.prepare_and_build(template=template, platform=Platform.ANDROID, change_set=change_set, result_file=android_result_file) - Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, + Helpers.prepare_and_build(template=template, platform=Platform.IOS, change_set=change_set, result_file=ios_result_file) @staticmethod diff --git a/tests/perf/tooling/build_perf_hello_world_js.py b/tests/perf/tooling/build_perf_hello_world_js.py index fb265c1d..028a45a1 100644 --- a/tests/perf/tooling/build_perf_hello_world_js.py +++ b/tests/perf/tooling/build_perf_hello_world_js.py @@ -3,8 +3,6 @@ import unittest -from parameterized import parameterized - from core.base_test.tns_test import TnsTest from core.enums.os_type import OSType from core.settings import Settings @@ -15,9 +13,8 @@ # noinspection PyMethodMayBeStatic,PyUnusedLocal class PrepareAndBuildPerfTests(TnsTest): - TEST_DATA = [ - ('hello-world-js', Template.HELLO_WORLD_JS.local_package, Changes.JSHelloWord.JS) - ] + template = Template.HELLO_WORLD_JS.local_package + change_set = Changes.JSHelloWord.JS @classmethod def setUpClass(cls): @@ -30,33 +27,26 @@ def setUp(self): def tearDownClass(cls): TnsTest.tearDownClass() - @parameterized.expand(TEST_DATA) - def test_001_prepare_data(self, template, template_package, change_set): - Helpers.prepare_data(template, template_package, change_set) + def test_001_prepare_data(self): + Helpers.prepare_data(self.template, self.change_set) - @parameterized.expand(TEST_DATA) - def test_200_prepare_android_initial(self, template, template_package, change_set): - Helpers.prepare_android_initial(template) + def test_200_prepare_android_initial(self): + Helpers.prepare_android_initial(self.template) - @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_201_prepare_ios_initial(self, template, template_package, change_set): - Helpers.prepare_ios_initial(template) + def test_201_prepare_ios_initial(self): + Helpers.prepare_ios_initial(self.template) - @parameterized.expand(TEST_DATA) - def test_300_build_android_initial(self, template, template_package, change_set): - Helpers.build_android_initial(template) + def test_300_build_android_initial(self): + Helpers.build_android_initial(self.template) - @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_301_build_ios_initial(self, template, template_package, change_set): - Helpers.build_ios_initial(template) + def test_301_build_ios_initial(self): + Helpers.build_ios_initial(self.template) - @parameterized.expand(TEST_DATA) - def test_310_build_android_incremental(self, template, template_package, change_set): - Helpers.build_android_incremental(template) + def test_310_build_android_incremental(self): + Helpers.build_android_incremental(self.template) - @parameterized.expand(TEST_DATA) @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_311_build_ios_incremental(self, template, template_package, change_set): - Helpers.build_ios_incremental(template) + def test_311_build_ios_incremental(self): + Helpers.build_ios_incremental(self.template)