diff --git a/.github/workflows/test-workflow-main.yml b/.github/workflows/test-workflow-main.yml index 43b73d1..54de3e9 100644 --- a/.github/workflows/test-workflow-main.yml +++ b/.github/workflows/test-workflow-main.yml @@ -53,6 +53,7 @@ jobs: validateAzd: false validatePaths: "README.md, .devcontainer, azure.yaml, NON_EXISTENT_FILE" expectedTopics: "ai-azd-templates, azd-templates" + validateTests: "Playwright" env: REPOSITORY_NAME: Azure-Samples/azd-ai-starter README_H2_TAGS: "## Features, ## Get Started" @@ -67,6 +68,7 @@ jobs: validatePaths: "None" expectedTopics: "None" useDevContainer: false + validateTests: "Playwright" env: REPOSITORY_NAME: Azure-Samples/azd-ai-starter diff --git a/.gitignore b/.gitignore index 82f9275..b453cfc 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# vscode +.vscode/ \ No newline at end of file diff --git a/action.yml b/action.yml index e3cea31..53dfb1c 100644 --- a/action.yml +++ b/action.yml @@ -25,6 +25,12 @@ inputs: default: 'PSRule' options: - 'PSRule' + validateTests: + description: 'Run tests for validation' + required: false + default: 'Playwright' + options: + - 'Playwright' outputs: resultFile: description: "A file path to a results file." @@ -273,6 +279,23 @@ runs: python3 -m pip install -r tva_${{ github.run_id }}/requirements.txt; subFolder: ${{ steps.reform_path.outputs.workingDirectory }} + - name: Setup Node in devcontainer + id: setup-node + if: ${{ inputs.useDevContainer == 'true' && inputs.validateTests == 'Playwright' }} + uses: devcontainers/ci@v0.3 + with: + runCmd: | + if ! command -v node &> /dev/null; then + echo "node not found, installing lts. node..." + curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - + sudo apt-get install -y nodejs + fi + echo "Install playwright node..." + npm install @playwright/test + echo "Install playwright browsers..." + npx playwright install --with-deps + subFolder: ${{ steps.reform_path.outputs.workingDirectory }} + - name: Remove spaces from inputs id: remove_spaces run: | @@ -300,6 +323,9 @@ runs: if [ ${{ inputs.securityAction }} == "PSRule" ]; then arguments+=" --psrule_result ./psrule-output.json" fi + if [ ${{ inputs.validateTests }} == "Playwright" ]; then + arguments+=" --validate_playwright_test" + fi echo "arguments=$arguments" >> $GITHUB_OUTPUT shell: bash @@ -326,7 +352,7 @@ runs: CREATE_ROLE_FOR_USER=false AZURE_PRINCIPAL_TYPE=ServicePrincipal FORCE_TERRAFORM_REMOTE_STATE_CREATION=false - + - name: Send output to main workflow id: send_output if: ${{ inputs.useDevContainer == 'true' }} @@ -340,6 +366,21 @@ runs: if: ${{ inputs.useDevContainer == 'false' }} uses: Azure/setup-azd@v2 + - uses: actions/setup-node@v4 + if: ${{ inputs.useDevContainer == 'false' && inputs.validateTests == 'Playwright' }} + with: + node-version: lts/* + + - name: Install playwright + if: ${{ inputs.useDevContainer == 'false' && inputs.validateTests == 'Playwright' }} + working-directory: ${{ inputs.workingDirectory }} + run: | + echo "Install playwright node..." + npm install @playwright/test + echo "Install playwright browsers..." + npx playwright install --with-deps + shell: bash + - name: Log in with Azure (Federated Credentials) if: ${{ inputs.validateAzd == 'true' && inputs.useDevContainer == 'false' }} run: | diff --git a/src/gallery_validate.py b/src/gallery_validate.py index 7518080..6c493d1 100644 --- a/src/gallery_validate.py +++ b/src/gallery_validate.py @@ -34,6 +34,9 @@ def main(): type=str, help="The output file path of PSRule.", ) + parser.add_argument( + "--validate_playwright_test", action="store_true", help="Run Playwright tests." + ) parser.add_argument("--output", type=str, help="The output file path.") parser.add_argument("--debug", action="store_true", help="Enable debug logging.") @@ -43,7 +46,7 @@ def main(): logging.basicConfig(format="%(message)s", level=log_level) logging.debug( - f"Repo path: {args.repo_path} validate_paths: {args.validate_paths} validate_azd: {args.validate_azd} debug: {args.debug} topics: {args.topics} expected_topics: {args.expected_topics} psrule: {args.psrule_result} output: {args.output}" + f"Repo path: {args.repo_path} validate_paths: {args.validate_paths} validate_azd: {args.validate_azd} debug: {args.debug} topics: {args.topics} expected_topics: {args.expected_topics} validate_playwright_test: {args.validate_playwright_test} psrule: {args.psrule_result} output: {args.output}" ) # Parse rules and generate validators diff --git a/src/rule_parser.py b/src/rule_parser.py index be4837f..f11331f 100644 --- a/src/rule_parser.py +++ b/src/rule_parser.py @@ -6,6 +6,7 @@ from validator.topic_validator import TopicValidator from validator.folder_validator import FolderValidator from validator.ps_rule_validator import PSRuleValidator +from validator.playwright_test_validator import PlaywrightTestValidator from validator.azd_command import AzdCommand from severity import Severity import utils @@ -143,6 +144,33 @@ def parse(self): elif validator_type == "PSRuleValidator": validator = PSRuleValidator(catalog, self.args.psrule_result, severity) validators.append(validator) + + elif validator_type == "PlaywrightTestValidator": + if not self.args.validate_playwright_test: + continue + + playwright_config_ts_paths = utils.find_playwright_config_ts_path( + self.args.repo_path + ) + logging.debug( + f"playwright_config_ts_paths: {playwright_config_ts_paths}" + ) + + for playwright_config_ts_path in playwright_config_ts_paths: + playwright_validator = PlaywrightTestValidator( + catalog, playwright_config_ts_path, severity + ) + inserted = False + for i, validator in enumerate(validators): + if ( + isinstance(validator, AzdValidator) + and validator.command == AzdCommand.UP + ): + validators.insert(i + 1, playwright_validator) + inserted = True + break + if not inserted: + validators.append(playwright_validator) else: continue diff --git a/src/rules.json b/src/rules.json index 1917b37..c50d8a9 100644 --- a/src/rules.json +++ b/src/rules.json @@ -102,5 +102,10 @@ "catalog": "security_requirements", "validator": "PSRuleValidator", "severity": "low" + }, + "playwright test": { + "catalog": "functional_requirements", + "validator": "PlaywrightTestValidator", + "severity": "low" } } diff --git a/src/utils.py b/src/utils.py index 676a3fd..93ddfbe 100644 --- a/src/utils.py +++ b/src/utils.py @@ -11,6 +11,15 @@ def find_infra_yaml_path(repo_path): return infra_yaml_paths if len(infra_yaml_paths) > 0 else [repo_path] +def find_playwright_config_ts_path(repo_path): + # playwright.config.ts + playwright_config_ts_paths = [] + for root, dirs, files in os.walk(repo_path): + if "playwright.config.ts" in files: + playwright_config_ts_paths.append(root) + return playwright_config_ts_paths if len(playwright_config_ts_paths) > 0 else [] + + def indent(text, count=2): return (" " * count).join(text.splitlines(True)) diff --git a/src/validator/playwright_test_validator.py b/src/validator/playwright_test_validator.py new file mode 100644 index 0000000..4037d83 --- /dev/null +++ b/src/validator/playwright_test_validator.py @@ -0,0 +1,74 @@ +import subprocess +import logging +from validator.validator_base import ValidatorBase +from constants import ItemResultFormat, line_delimiter, Signs +from utils import indent +from severity import Severity +import re + + +class PlaywrightTestValidator(ValidatorBase): + def __init__( + self, + validatorCatalog, + folderPath, + severity=Severity.LOW, + ): + super().__init__("PlaywrightTestValidator", validatorCatalog, severity) + self.folderPath = folderPath + + def validate(self): + self.result = True + self.messages = [] + + result, message = self.playwright_test() + self.result = self.result and result + self.messages.append(self.replace_words(self.escape_ansi(message), "|", "-")) + + self.resultMessage = line_delimiter.join(self.messages) + return self.result, self.resultMessage + + def playwright_test(self): + logging.debug(f"Running playwright test in {self.folderPath}") + return self.runCommand( + "npx playwright test", "--pass-with-no-tests --quiet --reporter list" + ) + + def escape_ansi(self, line): + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + logging.debug("Escaping ansi characters in line") + return ansi_escape.sub("", line) + + def replace_words(self, line, word, replacement): + logging.debug(f"removing {word} with {replacement} from line") + return line.replace(word, replacement) + + def runCommand(self, command, arguments): + message = ( + f"{command}" + if self.folderPath == "." + else f"{command} in {self.folderPath}" + ) + try: + result = subprocess.run( + " ".join([command, arguments]), + cwd=self.folderPath, + capture_output=True, + text=True, + check=True, + shell=True, + ) + logging.info(f"{result.stdout}") + return True, ItemResultFormat.PASS.format(message=message) + except subprocess.CalledProcessError as e: + logging.info(f"{e.stdout}") + logging.warning(f"{e.stderr}") + return False, ItemResultFormat.AZD_FAIL.format( + sign=Signs.BLOCK + if Severity.isBlocker(self.severity) + else Signs.WARNING, + message=message, + detail_messages=ItemResultFormat.DETAILS.format( + message=indent(e.stdout.replace("\\", "")) + ), + ) diff --git a/test/test_gallery_validate.py b/test/test_gallery_validate.py index a811555..3d2cf40 100644 --- a/test/test_gallery_validate.py +++ b/test/test_gallery_validate.py @@ -17,6 +17,7 @@ def test_main( repo_path="dummy_repo_path", validate_paths=None, validate_azd=True, + validate_playwright_test=True, topics="azd-templates,azure", expected_topics=None, msdoresult="dummy_msdo_result_file", diff --git a/test/test_rule_parser.py b/test/test_rule_parser.py index 65a3486..7c3631c 100644 --- a/test/test_rule_parser.py +++ b/test/test_rule_parser.py @@ -8,6 +8,7 @@ from validator.topic_validator import TopicValidator from validator.azd_command import AzdCommand from validator.ps_rule_validator import PSRuleValidator +from validator.playwright_test_validator import PlaywrightTestValidator from severity import Severity @@ -177,6 +178,145 @@ def test_parse_azd_validator(self, mock_file, mock_find_infra_yaml_path): self.assertEqual(azd_down_validator.command, AzdCommand.DOWN) self.assertEqual(azd_down_validator.severity, Severity.MODERATE) + @patch("utils.find_playwright_config_ts_path") + @patch("utils.find_infra_yaml_path") + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps( + { + "azd up": { + "catalog": "Functional Requirements", + "validator": "AzdValidator", + "severity": "high", + }, + "azd down": { + "catalog": "Functional Requirements", + "validator": "AzdValidator", + "severity": "moderate", + }, + "playwright test": { + "catalog": "Functional Requirements", + "validator": "PlaywrightTestValidator", + "severity": "low", + }, + } + ), + ) + def test_parse_mixed_azd_playwright_validator( + self, mock_file, find_infra_yaml_path, find_playwright_config_ts_path + ): + find_infra_yaml_path.return_value = ["mocked/path/to/infra.yaml"] + find_playwright_config_ts_path.return_value = [ + "mocked/path/to/playwright.config.ts" + ] + args = argparse.Namespace( + validate_azd=True, + validate_playwright_test=True, + topics=None, + repo_path=".", + validate_paths=None, + expected_topics=None, + ) + parser = RuleParser("dummy_path", args) + validators = parser.parse() + + self.assertEqual(len(validators), 3) + + azd_up_validator = validators[0] + self.assertIsInstance(azd_up_validator, AzdValidator) + self.assertEqual(azd_up_validator.catalog, "Functional Requirements") + self.assertEqual(azd_up_validator.folderPath, "mocked/path/to/infra.yaml") + self.assertEqual(azd_up_validator.command, AzdCommand.UP) + self.assertEqual(azd_up_validator.severity, Severity.HIGH) + + playwright_test_validator = validators[1] + self.assertIsInstance(playwright_test_validator, PlaywrightTestValidator) + self.assertEqual(playwright_test_validator.catalog, "Functional Requirements") + self.assertEqual( + playwright_test_validator.folderPath, "mocked/path/to/playwright.config.ts" + ) + self.assertEqual(playwright_test_validator.severity, Severity.LOW) + + azd_down_validator = validators[2] + self.assertIsInstance(azd_down_validator, AzdValidator) + self.assertEqual(azd_down_validator.catalog, "Functional Requirements") + self.assertEqual(azd_down_validator.folderPath, "mocked/path/to/infra.yaml") + self.assertEqual(azd_down_validator.command, AzdCommand.DOWN) + self.assertEqual(azd_down_validator.severity, Severity.MODERATE) + + @patch("utils.find_playwright_config_ts_path") + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps( + { + "playwright test": { + "catalog": "Functional Requirements", + "validator": "PlaywrightTestValidator", + "severity": "low", + } + } + ), + ) + def test_parse_playwright_validator( + self, mock_file, find_playwright_config_ts_path + ): + find_playwright_config_ts_path.return_value = [ + "mocked/path/to/playwright.config.ts" + ] + args = argparse.Namespace( + validate_azd=False, + validate_playwright_test=True, + topics=None, + repo_path=".", + validate_paths=None, + expected_topics=None, + ) + parser = RuleParser("dummy_path", args) + validators = parser.parse() + + self.assertEqual(len(validators), 1) + + playwright_test_validator = validators[0] + self.assertIsInstance(playwright_test_validator, PlaywrightTestValidator) + self.assertEqual(playwright_test_validator.catalog, "Functional Requirements") + self.assertEqual( + playwright_test_validator.folderPath, "mocked/path/to/playwright.config.ts" + ) + self.assertEqual(playwright_test_validator.severity, Severity.LOW) + + @patch("utils.find_playwright_config_ts_path") + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps( + { + "playwright test": { + "catalog": "Functional Requirements", + "validator": "PlaywrightTestValidator", + "severity": "low", + } + } + ), + ) + def test_parse_playwright_validator_empty( + self, mock_file, find_playwright_config_ts_path + ): + find_playwright_config_ts_path.return_value = [] + args = argparse.Namespace( + validate_azd=False, + validate_playwright_test=True, + topics=None, + repo_path=".", + validate_paths=None, + expected_topics=None, + ) + parser = RuleParser("dummy_path", args) + validators = parser.parse() + + self.assertEqual(len(validators), 0) + @patch( "builtins.open", new_callable=mock_open,