From 5c71801ee4a3e968bf1f73978885bb745d918c13 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 Aug 2025 09:11:00 +0000 Subject: [PATCH 01/19] Initial plan From 711d78380b5b15a79703c7b2deb1057c6ebd9847 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 Aug 2025 09:30:36 +0000 Subject: [PATCH 02/19] Implement MySQL live tests and integrate into functional environment Co-authored-by: Archmonger <16909269+Archmonger@users.noreply.github.com> --- pyproject.toml | 4 +- scripts/mysql_live_test.py | 384 +++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 scripts/mysql_live_test.py diff --git a/pyproject.toml b/pyproject.toml index bb10f605..42e5fd92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,7 +175,7 @@ extra-dependencies = [ "testfixtures", "django", "psycopg2-binary", - # "mysqlclient", + "mysqlclient", ] [tool.hatch.envs.functional.env-vars] @@ -190,9 +190,11 @@ DB_NAME = "tmp/test_db.sqlite3" all = [ "hatch run functional:sqlite --all {args}", "hatch run functional:postgres --all {args}", + "hatch run functional:mysql --all {args}", ] sqlite = ["python scripts/sqlite_live_test.py {args}"] postgres = ["python scripts/postgres_live_test.py {args}"] +mysql = ["python scripts/mysql_live_test.py {args}"] # >>> Generic Tools <<< diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py new file mode 100644 index 00000000..78b101a5 --- /dev/null +++ b/scripts/mysql_live_test.py @@ -0,0 +1,384 @@ +"""MySQL Live Functional Test Script for django-dbbackup + +Usage: + python scripts/mysql_live_test.py [--verbose] + python scripts/mysql_live_test.py --connector MysqlDumpConnector + python scripts/mysql_live_test.py --all + +It provides end-to-end validation of MySQL backup/restore functionality using the +available connectors and mirrors the visual layout & summary style of the SQLite and +PostgreSQL live tests for consistency. + +Exit code 0 on success (all tested connectors passed), 1 on failure. +""" + +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +# Add parent directory to path to import Django modules +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from scripts._utils import get_symbols + +_SYMS = get_symbols() +SYMBOL_PASS = _SYMS["PASS"] +SYMBOL_FAIL = _SYMS["FAIL"] +SYMBOL_SUMMARY = _SYMS["SUMMARY"] +SYMBOL_MYSQL = "🐬" # MySQL dolphin emoji +SYMBOL_TEST = _SYMS["TEST"] + +# Available MySQL connectors +MYSQL_CONNECTORS = [ + "MysqlDumpConnector", +] + +import django +from django.core.management import execute_from_command_line + +GITHUB_ACTIONS: bool = os.getenv("GITHUB_ACTIONS", "false").lower() in ("true", "1", "yes") + + +class MySQLTestRunner: + """Manages a test database on the existing MySQL instance.""" + + def __init__(self, verbose=False): + self.verbose = verbose + self.temp_dir = Path(tempfile.mkdtemp(prefix="mysql_live_test_")) + self.test_db_name = f"dbbackup_test_{int(time.time())}" + self.user = "dbbackup_test_user" + self.password = "test_password_123" + self.host = "localhost" + self.port = 3306 + self.superuser = "root" + self.db_created = False + self.user_created = False + + def _log(self, message): + if self.verbose: + print(f"[MySQL Test] {message}") + + def _run_command(self, cmd, capture_output=False, use_sudo=False): + """Run a command and return stdout.""" + if use_sudo and not GITHUB_ACTIONS: + # For local development, might need sudo for MySQL operations + cmd = ["sudo"] + cmd + + self._log(f"Running: {' '.join(cmd)}") + + if capture_output: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Command failed: {result.stderr}") + return result.stdout.strip() + else: + result = subprocess.run(cmd) + if result.returncode != 0: + raise RuntimeError(f"Command failed with exit code {result.returncode}") + + def _create_test_database(self): + """Create the test database.""" + self._log(f"Creating test database: {self.test_db_name}") + + # Try different MySQL authentication methods + mysql_commands = [ + ["sudo", "mysql"], # auth_socket (common on Ubuntu) + ["mysql", "-u", "root"], # no password + ["mysql", "-u", "root", "-p"], # password prompt (will fail in automation) + ] + + if GITHUB_ACTIONS: + self._log("GitHub Actions detected, using root user as database owner") + # In CI, try the standard root access + mysql_commands = [ + ["mysql", "-u", "root"], + ["mysql", "-u", "root", "-h", "localhost"], + ] + self.user = self.superuser + self.password = "" # No password for root in CI typically + else: + # For local development, use sudo mysql (auth_socket) + self.user = self.superuser # Use root for simplicity + self.password = "" + + # Try to connect and create database + create_db_sql = f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};" + + for mysql_cmd in mysql_commands: + try: + cmd = mysql_cmd + ["-e", create_db_sql] + self._log(f"Trying MySQL connection with: {' '.join(mysql_cmd)}") + self._run_command(cmd, capture_output=True) + self._log("Successfully connected to MySQL and created database") + + # Test the connection works + test_cmd = mysql_cmd + ["-e", "SELECT 1;"] + self._run_command(test_cmd, capture_output=True) + + # Store the working command for later use + self.mysql_base_cmd = mysql_cmd + self.db_created = True + return + + except RuntimeError as e: + self._log(f"MySQL connection failed with {' '.join(mysql_cmd)}: {e}") + continue + + # If all methods fail, raise an error + raise RuntimeError("Could not connect to MySQL with any authentication method. " + "Please ensure MySQL is running and accessible.") + + def get_database_config(self): + """Return Django database configuration.""" + return { + "ENGINE": "django.db.backends.mysql", + "NAME": self.test_db_name, + "USER": self.user, + "PASSWORD": self.password, + "HOST": self.host, + "PORT": self.port, + "OPTIONS": { + "init_command": "SET sql_mode='STRICT_TRANS_TABLES'", + }, + } + + def cleanup(self): + """Clean up test database and user.""" + if self.db_created and hasattr(self, 'mysql_base_cmd'): + try: + self._log(f"Dropping test database: {self.test_db_name}") + cmd = self.mysql_base_cmd + ["-e", f"DROP DATABASE IF EXISTS {self.test_db_name};"] + self._run_command(cmd) + except Exception as e: + self._log(f"Failed to drop database: {e}") + + if self.user_created and hasattr(self, 'mysql_base_cmd') and not GITHUB_ACTIONS: + try: + self._log(f"Dropping test user: {self.user}") + cmd = self.mysql_base_cmd + ["-e", f"DROP USER IF EXISTS '{self.user}'@'localhost';"] + self._run_command(cmd) + except Exception as e: + self._log(f"Failed to drop user: {e}") + + # Clean up temp directory + if self.temp_dir.exists(): + shutil.rmtree(self.temp_dir) + self._log(f"Removed temp directory: {self.temp_dir}") + + +class MySQLLiveTest: + """Runs live tests against MySQL connectors.""" + + def __init__(self, connector_name, verbose=False): + self.connector_name = connector_name + self.verbose = verbose + self.mysql_runner = MySQLTestRunner(verbose=verbose) + + def _log(self, message): + if self.verbose: + print(f"[MySQL Live Test] {message}") + + def _configure_django(self): + """Configure Django with the test MySQL database.""" + # Configure Django settings + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") + + # Override database settings + db_config = self.mysql_runner.get_database_config() + os.environ.update({ + "DB_ENGINE": db_config["ENGINE"], + "DB_NAME": db_config["NAME"], + "DB_USER": db_config["USER"], + "DB_HOST": db_config["HOST"], + }) + # Only set password if it exists + if db_config["PASSWORD"]: + os.environ["DB_PASSWORD"] = db_config["PASSWORD"] + # Set port as string + os.environ["DB_PORT"] = str(db_config["PORT"]) + + # Set connector + os.environ["CONNECTOR"] = f"dbbackup.db.mysql.{self.connector_name}" + + # Configure storage for backups - use unique directory per test + backup_dir = os.path.join(str(self.mysql_runner.temp_dir), "backups") + os.makedirs(backup_dir, exist_ok=True) + os.environ.update({ + "STORAGE": "django.core.files.storage.FileSystemStorage", + "STORAGE_LOCATION": backup_dir, + "STORAGE_OPTIONS": f"location={backup_dir}", + "MEDIA_ROOT": os.path.join(str(self.mysql_runner.temp_dir), "media"), + }) + + # Initialize Django + django.setup() + + def _create_test_data(self): + """Create test data in the database.""" + self._log("Creating test data...") + + # Run migrations + execute_from_command_line(["", "migrate", "--noinput"]) + + # Create test models + from tests.testapp.models import CharModel, TextModel + + # Create some test data (CharModel has max_length=10) + char_obj = CharModel.objects.create(field="test_char") # 9 chars, fits in 10 + text_obj = TextModel.objects.create(field="test text content for backup") + + self._log(f"Created CharModel: {char_obj}") + self._log(f"Created TextModel: {text_obj}") + + return char_obj, text_obj + + def _verify_test_data(self, expected_char_count, expected_text_count): + """Verify that the expected test data exists.""" + from tests.testapp.models import CharModel, TextModel + + actual_char_count = CharModel.objects.count() + actual_text_count = TextModel.objects.count() + + self._log(f"Data verification - CharModel: {actual_char_count}/{expected_char_count}, TextModel: {actual_text_count}/{expected_text_count}") + + if actual_char_count != expected_char_count: + raise AssertionError(f"CharModel count mismatch: expected {expected_char_count}, got {actual_char_count}") + if actual_text_count != expected_text_count: + raise AssertionError(f"TextModel count mismatch: expected {expected_text_count}, got {actual_text_count}") + + return True + + def run_test(self): + """Run the full backup/restore test cycle.""" + try: + self._log(f"Starting MySQL live test for {self.connector_name}") + + # 1. Setup MySQL database + self.mysql_runner._create_test_database() + + # 2. Configure Django + self._configure_django() + + # 3. Create test data + char_obj, text_obj = self._create_test_data() + expected_char_count = 1 + expected_text_count = 1 + + # 4. Verify initial data + self._verify_test_data(expected_char_count, expected_text_count) + + # 5. Create database backup + self._log("Creating database backup...") + execute_from_command_line(["", "dbbackup", "--noinput"]) + + # 6. Create media files and backup + media_dir = os.environ["MEDIA_ROOT"] + os.makedirs(media_dir, exist_ok=True) + + # Create test media files + test_file = os.path.join(media_dir, "test.txt") + with open(test_file, "w") as f: + f.write("test content") + + self._log("Creating media backup...") + execute_from_command_line(["", "mediabackup", "--noinput"]) + + # 7. Clear the database + self._log("Clearing database for restore test...") + from tests.testapp.models import CharModel, TextModel + CharModel.objects.all().delete() + TextModel.objects.all().delete() + + # Remove media files + if os.path.exists(test_file): + os.remove(test_file) + + # 8. Restore database + self._log("Restoring database backup...") + execute_from_command_line(["", "dbrestore", "--noinput"]) + + # 9. Restore media + self._log("Restoring media backup...") + execute_from_command_line(["", "mediarestore", "--noinput"]) + + # 10. Verify restored data + self._verify_test_data(expected_char_count, expected_text_count) + + # 11. Verify restored media + if not os.path.exists(test_file): + raise AssertionError(f"Media file not restored: {test_file}") + + with open(test_file, "r") as f: + content = f.read() + if content != "test content": + raise AssertionError(f"Media file content mismatch: expected 'test content', got '{content}'") + + self._log(f"MySQL live test for {self.connector_name} completed successfully") + return True + + except Exception as e: + self._log(f"MySQL live test for {self.connector_name} failed: {e}") + return False + finally: + # Cleanup + self.mysql_runner.cleanup() + + +def _run_all(connectors, verbose: bool) -> int: + """Run tests for all connectors.""" + overall_success = True + results = {} + + for name in connectors: + cmd = [sys.executable, __file__, "--connector", name] + if verbose: + cmd.append("-v") + + print(f"\n{SYMBOL_TEST} Testing {name}...") + proc = subprocess.run(cmd, check=False) + passed = proc.returncode == 0 + results[name] = passed + status = f"{SYMBOL_PASS} PASSED" if passed else f"{SYMBOL_FAIL} FAILED" + print(f" {name}: {status}") + overall_success &= passed + + print(f"\n{SYMBOL_SUMMARY} MySQL Connector Test Summary") + for name, passed in results.items(): + status = SYMBOL_PASS if passed else SYMBOL_FAIL + print(f" {status} {name}") + + return 0 if overall_success else 1 + + +def main() -> int: + """Main entry point for MySQL live tests.""" + import argparse + + parser = argparse.ArgumentParser(description="Run live MySQL functional tests for django-dbbackup") + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output") + parser.add_argument( + "--connector", + choices=MYSQL_CONNECTORS, + default="MysqlDumpConnector", + help="MySQL connector to test (default: %(default)s)", + ) + parser.add_argument("--all", action="store_true", help="Test all MySQL connectors") + + args = parser.parse_args() + verbose = args.verbose + + if args.all: + return _run_all(MYSQL_CONNECTORS, verbose) + + # Run single connector test + test = MySQLLiveTest(args.connector, verbose=verbose) + success = test.run_test() + return 0 if success else 1 + + +if __name__ == "__main__": # pragma: no cover - executed as script + sys.exit(main()) \ No newline at end of file From d7d477d5869d3134ac244b785d99d31cca6568dd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 Aug 2025 09:36:22 +0000 Subject: [PATCH 03/19] Add MySQL CI integration and update documentation Co-authored-by: Archmonger <16909269+Archmonger@users.noreply.github.com> --- .github/copilot-instructions.md | 24 ++++++++++++----- .github/workflows/ci.yml | 6 +++++ scripts/_utils.py | 7 +++-- scripts/mysql_live_test.py | 47 ++++++++++++++++++++++++++++----- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 53c72346..b8eab299 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -59,6 +59,14 @@ Always test backup and restore functionality after making changes using function hatch run functional:all ``` +2. **Individual Backend Tests:** + + ```bash + hatch run functional:sqlite --all # SQLite connectors + hatch run functional:postgres --all # PostgreSQL connectors + hatch run functional:mysql --all # MySQL connectors + ``` + 2. **Manual Database Test (if needed):** ```bash @@ -85,6 +93,7 @@ Always test backup and restore functionality after making changes using function ```bash hatch run functional:sqlite --all hatch run functional:postgres --all + hatch run functional:mysql --all ``` ## Troubleshooting Known Issues @@ -102,7 +111,9 @@ Modern development process using Hatch: 1. **Bootstrap environment**: `pip install --upgrade pip hatch uv` 2. **Make your changes** to the codebase 3. **Run unit tests**: `hatch test` (β‰ˆ30s) **All must pass - failures are never expected or allowed.** -4. **Run functional tests**: `hatch run functional:all` (β‰ˆ10–15s) +4. **Run functional tests**: `hatch run functional:all` (β‰ˆ15–25s) + - Includes SQLite, PostgreSQL, and MySQL live tests + - MySQL tests are automatically skipped if MySQL server is not available 5. **Run linting**: `hatch run lint:check` (5 seconds) 6. **Auto-format code**: `hatch run lint:format` (2 seconds) 7. **Test documentation**: `hatch run docs:build` (2 seconds) @@ -127,7 +138,7 @@ Key directories and files: - `management/commands/` – Django management commands (dbbackup, dbrestore, mediabackup, mediarestore, listbackups) - Core modules: `storage.py`, `settings.py`, `log.py`, `signals.py`, `utils.py` - `tests/` – comprehensive test suite (unit + helpers) -- `scripts/` – live functional test scripts (`sqlite_live_test.py`, `postgres_live_test.py`) +- `scripts/` – live functional test scripts (`sqlite_live_test.py`, `postgres_live_test.py`, `mysql_live_test.py`) - `docs/` – MkDocs Material source (built site output under `docs/site/` when building locally) - `pyproject.toml` – project + Hatch environment configuration - `.github/workflows/ci.yml` – CI matrix & publish pipeline @@ -152,7 +163,8 @@ hatch test # Run all tests across matrix hatch test --python 3.12 # Test specific Python version subset hatch run functional:all # Functional tests (SQLite + PostgreSQL) hatch run functional:sqlite --all # Functional tests (SQLite only) -hatch run functional:postgres --all # Functional tests (PostgreSQL only) +hatch run functional:postgres --all # Functional tests (PostgreSQL only) +hatch run functional:mysql --all # Functional tests (MySQL only) hatch run lint:check # Lint (ruff + pylint) hatch run lint:format # Auto-format hatch run lint:format-check # Format check only @@ -181,7 +193,7 @@ Modern isolated environments configured in pyproject.toml: ### Testing Environments - **hatch-test**: Unit testing (Python 3.9–3.13 Γ— Django 4.2/5.0/5.1/5.2 matrix) -- **functional**: End-to-end backup/restore (filesystem storage; live SQLite & PostgreSQL scripts) +- **functional**: End-to-end backup/restore (filesystem storage; live SQLite, PostgreSQL & MySQL scripts) ### Development Environments @@ -225,7 +237,7 @@ Core runtime dependency: Development dependencies (managed by hatch): -- **Testing**: coverage, django-storages, psycopg2-binary, python-gnupg, testfixtures, python-dotenv +- **Testing**: coverage, django-storages, psycopg2-binary, mysqlclient, python-gnupg, testfixtures, python-dotenv - **Linting**: ruff, pylint - **Documentation**: mkdocs, mkdocs-material, mkdocs-git-revision-date-localized-plugin, mkdocs-include-markdown-plugin, mkdocs-spellcheck[all], mkdocs-git-authors-plugin, mkdocs-minify-plugin, mike, linkcheckmd - **Pre-commit**: pre-commit @@ -236,7 +248,7 @@ Modern GitHub Actions workflow (.github/workflows/build.yml): - **Lint Python**: Code quality checks (temporarily set to pass) - **Test Python**: Matrix testing across Python 3.9-3.13 with coverage -- **Functional Tests**: End-to-end backup/restore verification (SQLite + PostgreSQL live scripts) +- **Functional Tests**: End-to-end backup/restore verification (SQLite, PostgreSQL & MySQL live scripts) - **Coverage**: Artifact-based coverage combining with 80% threshold - **Build**: Package building with hatch - **Publish GitHub**: Automated GitHub release creation on tags diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4308df5..f2304d5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,12 @@ jobs: - run: psql -c "SELECT 1" env: PGSERVICE: postgres + - name: Setup MySQL + if: matrix.os == 'ubuntu-latest' + run: | + sudo systemctl start mysql + sudo mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '';" + mysql -u root -e "SELECT VERSION();" - name: Run functional tests run: hatch run functional:all -v diff --git a/scripts/_utils.py b/scripts/_utils.py index c89ec666..9cdbcf9a 100644 --- a/scripts/_utils.py +++ b/scripts/_utils.py @@ -11,9 +11,10 @@ print(SYMS['PASS'], 'Test passed') Provided symbol keys: - PASS, FAIL, SUMMARY, TEST, PG + PASS, FAIL, SUMMARY, TEST, PG, MYSQL -The ``PG`` key is only used by the PostgreSQL live test; others are shared. +The ``PG`` key is only used by the PostgreSQL live test; ``MYSQL`` is only +used by the MySQL live test; others are shared. """ from __future__ import annotations @@ -27,6 +28,7 @@ "SUMMARY": "πŸ“Š", "TEST": "πŸ“‹", "PG": "🐘", + "MYSQL": "🐬", } _ASCII_SYMBOLS = { @@ -35,6 +37,7 @@ "SUMMARY": "", "TEST": "", "PG": "", + "MYSQL": "", } diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index 78b101a5..a0c12727 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -29,7 +29,7 @@ SYMBOL_PASS = _SYMS["PASS"] SYMBOL_FAIL = _SYMS["FAIL"] SYMBOL_SUMMARY = _SYMS["SUMMARY"] -SYMBOL_MYSQL = "🐬" # MySQL dolphin emoji +SYMBOL_MYSQL = _SYMS["MYSQL"] # Use the symbol from _utils SYMBOL_TEST = _SYMS["TEST"] # Available MySQL connectors @@ -320,6 +320,13 @@ def run_test(self): self._log(f"MySQL live test for {self.connector_name} completed successfully") return True + except RuntimeError as e: + if "Could not connect to MySQL" in str(e): + self._log(f"MySQL not available, skipping test: {e}") + return "skipped" + else: + self._log(f"MySQL live test for {self.connector_name} failed: {e}") + return False except Exception as e: self._log(f"MySQL live test for {self.connector_name} failed: {e}") return False @@ -332,6 +339,7 @@ def _run_all(connectors, verbose: bool) -> int: """Run tests for all connectors.""" overall_success = True results = {} + skipped_count = 0 for name in connectors: cmd = [sys.executable, __file__, "--connector", name] @@ -340,17 +348,36 @@ def _run_all(connectors, verbose: bool) -> int: print(f"\n{SYMBOL_TEST} Testing {name}...") proc = subprocess.run(cmd, check=False) - passed = proc.returncode == 0 + + if proc.returncode == 0: + passed = True + status = f"{SYMBOL_PASS} PASSED" + elif proc.returncode == 2: # Special exit code for skipped + passed = "skipped" + status = f"⏭️ SKIPPED" + skipped_count += 1 + else: + passed = False + status = f"{SYMBOL_FAIL} FAILED" + overall_success = False + results[name] = passed - status = f"{SYMBOL_PASS} PASSED" if passed else f"{SYMBOL_FAIL} FAILED" print(f" {name}: {status}") - overall_success &= passed print(f"\n{SYMBOL_SUMMARY} MySQL Connector Test Summary") for name, passed in results.items(): - status = SYMBOL_PASS if passed else SYMBOL_FAIL + if passed == "skipped": + status = "⏭️ " + elif passed: + status = SYMBOL_PASS + else: + status = SYMBOL_FAIL print(f" {status} {name}") + if skipped_count == len(connectors): + print(" ℹ️ All tests skipped (MySQL not available)") + return 0 # Don't fail if MySQL is not available + return 0 if overall_success else 1 @@ -376,8 +403,14 @@ def main() -> int: # Run single connector test test = MySQLLiveTest(args.connector, verbose=verbose) - success = test.run_test() - return 0 if success else 1 + result = test.run_test() + + if result == "skipped": + return 2 # Special exit code for skipped + elif result: + return 0 + else: + return 1 if __name__ == "__main__": # pragma: no cover - executed as script From 2a6a7e02e3ebdbf7186dfcecac50df0348b6f8bc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 Aug 2025 09:45:16 +0000 Subject: [PATCH 04/19] Update CI to use setup-mysql action and refactor MySQL live tests to match PostgreSQL pattern Co-authored-by: Archmonger <16909269+Archmonger@users.noreply.github.com> --- .github/workflows/ci.yml | 8 +- scripts/mysql_live_test.py | 327 ++++++++++++++++++++++++------------- 2 files changed, 215 insertions(+), 120 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2304d5b..904c97b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,10 +116,10 @@ jobs: PGSERVICE: postgres - name: Setup MySQL if: matrix.os == 'ubuntu-latest' - run: | - sudo systemctl start mysql - sudo mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '';" - mysql -u root -e "SELECT VERSION();" + uses: kayqueGovetri/setup-mysql@v3.0.0 + with: + mysql-version: "8.0" + mysql-root-password: "root" - name: Run functional tests run: hatch run functional:all -v diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index a0c12727..4d21c4b3 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -18,25 +18,26 @@ import sys import tempfile import time +from multiprocessing import Process from pathlib import Path -# Add parent directory to path to import Django modules -sys.path.insert(0, str(Path(__file__).parent.parent)) - from scripts._utils import get_symbols _SYMS = get_symbols() SYMBOL_PASS = _SYMS["PASS"] SYMBOL_FAIL = _SYMS["FAIL"] SYMBOL_SUMMARY = _SYMS["SUMMARY"] -SYMBOL_MYSQL = _SYMS["MYSQL"] # Use the symbol from _utils +SYMBOL_MYSQL = _SYMS["MYSQL"] SYMBOL_TEST = _SYMS["TEST"] -# Available MySQL connectors +# Available MySQL connectors MYSQL_CONNECTORS = [ "MysqlDumpConnector", ] +# Add parent directory to path to import Django modules +sys.path.insert(0, str(Path(__file__).parent.parent)) + import django from django.core.management import execute_from_command_line @@ -48,7 +49,7 @@ class MySQLTestRunner: def __init__(self, verbose=False): self.verbose = verbose - self.temp_dir = Path(tempfile.mkdtemp(prefix="mysql_live_test_")) + self.temp_dir = None self.test_db_name = f"dbbackup_test_{int(time.time())}" self.user = "dbbackup_test_user" self.password = "test_password_123" @@ -59,49 +60,75 @@ def __init__(self, verbose=False): self.user_created = False def _log(self, message): + """Log a message if verbose mode is enabled.""" if self.verbose: print(f"[MySQL Test] {message}") - def _run_command(self, cmd, capture_output=False, use_sudo=False): - """Run a command and return stdout.""" - if use_sudo and not GITHUB_ACTIONS: - # For local development, might need sudo for MySQL operations - cmd = ["sudo"] + cmd - + def _run_command(self, cmd, check=True, capture_output=False, **kwargs): + """Run a command and optionally check for errors.""" self._log(f"Running: {' '.join(cmd)}") - if capture_output: - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - raise RuntimeError(f"Command failed: {result.stderr}") - return result.stdout.strip() - else: - result = subprocess.run(cmd) - if result.returncode != 0: - raise RuntimeError(f"Command failed with exit code {result.returncode}") + result = subprocess.run(cmd, capture_output=capture_output, text=True, **kwargs) + if check and result.returncode != 0: + error_msg = f"Command failed with exit code {result.returncode}: {' '.join(cmd)}" + if capture_output: + if result.stdout: + error_msg += f"\nSTDOUT: {result.stdout}" + if result.stderr: + error_msg += f"\nSTDERR: {result.stderr}" + raise RuntimeError(error_msg) + return result + + def setup_mysql(self): + """Set up a test database on the existing MySQL instance.""" + + if not shutil.which("mysql") or not shutil.which("mysqldump"): + install_instructions = "" + if os.name == "posix": + install_instructions = ( + "\nInstall by running 'sudo apt install mysql-client mysql-server'\n" + "... then run 'sudo service mysql start' to start the server." + ) + elif os.name == "nt": + install_instructions = ( + "\nInstall MySQL from https://dev.mysql.com/downloads/installer/ " + "and ensure mysql and mysqldump are in your PATH." + ) + raise RuntimeError(f"MySQL client tools (mysql, mysqldump, etc) are not installed!{install_instructions}") + + self._log("Setting up test database...") + self.temp_dir = tempfile.mkdtemp(prefix="dbbackup_mysql_") + try: + # Create test database + self._create_test_database() + + except Exception as e: + self.cleanup() + raise RuntimeError(f"Failed to set up MySQL: {e}") from e def _create_test_database(self): """Create the test database.""" self._log(f"Creating test database: {self.test_db_name}") # Try different MySQL authentication methods - mysql_commands = [ - ["sudo", "mysql"], # auth_socket (common on Ubuntu) - ["mysql", "-u", "root"], # no password - ["mysql", "-u", "root", "-p"], # password prompt (will fail in automation) - ] + mysql_commands = [] if GITHUB_ACTIONS: - self._log("GitHub Actions detected, using root user as database owner") - # In CI, try the standard root access + self._log("GitHub Actions detected, using setup-mysql action configuration") + # GitHub Actions with setup-mysql action uses root user with password 'root' mysql_commands = [ - ["mysql", "-u", "root"], - ["mysql", "-u", "root", "-h", "localhost"], + ["mysql", "-u", "root", "-proot", "-h", "localhost"], + ["mysql", "-u", "root", "-proot"], ] self.user = self.superuser - self.password = "" # No password for root in CI typically + self.password = "root" # setup-mysql action default else: - # For local development, use sudo mysql (auth_socket) + # For local development, try various authentication methods + mysql_commands = [ + ["sudo", "mysql"], # auth_socket (common on Ubuntu) + ["mysql", "-u", "root"], # no password + ["mysql", "-u", "root", "-p"], # password prompt (will fail in automation) + ] self.user = self.superuser # Use root for simplicity self.password = "" @@ -111,7 +138,7 @@ def _create_test_database(self): for mysql_cmd in mysql_commands: try: cmd = mysql_cmd + ["-e", create_db_sql] - self._log(f"Trying MySQL connection with: {' '.join(mysql_cmd)}") + self._log(f"Trying MySQL connection with: {' '.join(mysql_cmd[:3])}") # Don't log password self._run_command(cmd, capture_output=True) self._log("Successfully connected to MySQL and created database") @@ -125,7 +152,7 @@ def _create_test_database(self): return except RuntimeError as e: - self._log(f"MySQL connection failed with {' '.join(mysql_cmd)}: {e}") + self._log(f"MySQL connection failed with {' '.join(mysql_cmd[:3])}: {e}") continue # If all methods fail, raise an error @@ -133,7 +160,7 @@ def _create_test_database(self): "Please ensure MySQL is running and accessible.") def get_database_config(self): - """Return Django database configuration.""" + """Get Django database configuration for the test MySQL instance.""" return { "ENGINE": "django.db.backends.mysql", "NAME": self.test_db_name, @@ -147,40 +174,41 @@ def get_database_config(self): } def cleanup(self): - """Clean up test database and user.""" + """Clean up the test database.""" + self._log("Cleaning up test database...") + if self.db_created and hasattr(self, 'mysql_base_cmd'): try: self._log(f"Dropping test database: {self.test_db_name}") cmd = self.mysql_base_cmd + ["-e", f"DROP DATABASE IF EXISTS {self.test_db_name};"] - self._run_command(cmd) + self._run_command(cmd, check=False) except Exception as e: - self._log(f"Failed to drop database: {e}") + self._log(f"Warning: Failed to drop test database: {e}") if self.user_created and hasattr(self, 'mysql_base_cmd') and not GITHUB_ACTIONS: try: self._log(f"Dropping test user: {self.user}") cmd = self.mysql_base_cmd + ["-e", f"DROP USER IF EXISTS '{self.user}'@'localhost';"] - self._run_command(cmd) + self._run_command(cmd, check=False) except Exception as e: - self._log(f"Failed to drop user: {e}") + self._log(f"Warning: Failed to drop test user: {e}") - # Clean up temp directory - if self.temp_dir.exists(): + if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) - self._log(f"Removed temp directory: {self.temp_dir}") class MySQLLiveTest: """Runs live tests against MySQL connectors.""" - def __init__(self, connector_name, verbose=False): + def __init__(self, connector_name="MysqlDumpConnector", verbose=False): self.connector_name = connector_name self.verbose = verbose self.mysql_runner = MySQLTestRunner(verbose=verbose) def _log(self, message): + """Log a message if verbose mode is enabled.""" if self.verbose: - print(f"[MySQL Live Test] {message}") + print(f"[Live Test] {message}") def _configure_django(self): """Configure Django with the test MySQL database.""" @@ -214,8 +242,9 @@ def _configure_django(self): "MEDIA_ROOT": os.path.join(str(self.mysql_runner.temp_dir), "media"), }) - # Initialize Django - django.setup() + # Setup Django only if not already configured + if not django.apps.apps.ready: + django.setup() def _create_test_data(self): """Create test data in the database.""" @@ -236,46 +265,55 @@ def _create_test_data(self): return char_obj, text_obj - def _verify_test_data(self, expected_char_count, expected_text_count): - """Verify that the expected test data exists.""" + def _verify_test_data(self, expected_char_obj, expected_text_obj): + """Verify that test data exists and matches expectations.""" from tests.testapp.models import CharModel, TextModel - actual_char_count = CharModel.objects.count() - actual_text_count = TextModel.objects.count() + char_objs = CharModel.objects.all() + text_objs = TextModel.objects.all() - self._log(f"Data verification - CharModel: {actual_char_count}/{expected_char_count}, TextModel: {actual_text_count}/{expected_text_count}") + self._log(f"Found {char_objs.count()} CharModel objects") + self._log(f"Found {text_objs.count()} TextModel objects") - if actual_char_count != expected_char_count: - raise AssertionError(f"CharModel count mismatch: expected {expected_char_count}, got {actual_char_count}") - if actual_text_count != expected_text_count: - raise AssertionError(f"TextModel count mismatch: expected {expected_text_count}, got {actual_text_count}") + if char_objs.count() != 1 or text_objs.count() != 1: + raise AssertionError( + f"Expected 1 of each model, found {char_objs.count()} CharModel and {text_objs.count()} TextModel" + ) + + char_obj = char_objs.first() + text_obj = text_objs.first() + + if char_obj.field != expected_char_obj.field: + raise AssertionError( + f"CharModel field mismatch: expected '{expected_char_obj.field}', got '{char_obj.field}'" + ) + + if text_obj.field != expected_text_obj.field: + raise AssertionError( + f"TextModel field mismatch: expected '{expected_text_obj.field}', got '{text_obj.field}'" + ) + + self._log("Test data verification passed") + + def run_backup_restore_test(self): + """Run a complete backup and restore test cycle.""" + self._log(f"Starting backup/restore test with {self.connector_name}") - return True - - def run_test(self): - """Run the full backup/restore test cycle.""" try: - self._log(f"Starting MySQL live test for {self.connector_name}") + # Setup MySQL + self.mysql_runner.setup_mysql() - # 1. Setup MySQL database - self.mysql_runner._create_test_database() - - # 2. Configure Django + # Configure Django self._configure_django() - # 3. Create test data + # Create test data char_obj, text_obj = self._create_test_data() - expected_char_count = 1 - expected_text_count = 1 - - # 4. Verify initial data - self._verify_test_data(expected_char_count, expected_text_count) - # 5. Create database backup - self._log("Creating database backup...") + # Run backup + self._log("Running database backup...") execute_from_command_line(["", "dbbackup", "--noinput"]) - # 6. Create media files and backup + # Create media files and backup media_dir = os.environ["MEDIA_ROOT"] os.makedirs(media_dir, exist_ok=True) @@ -284,12 +322,13 @@ def run_test(self): with open(test_file, "w") as f: f.write("test content") - self._log("Creating media backup...") + self._log("Running media backup...") execute_from_command_line(["", "mediabackup", "--noinput"]) - # 7. Clear the database - self._log("Clearing database for restore test...") + # Clear test data + self._log("Clearing test data...") from tests.testapp.models import CharModel, TextModel + CharModel.objects.all().delete() TextModel.objects.all().delete() @@ -297,18 +336,24 @@ def run_test(self): if os.path.exists(test_file): os.remove(test_file) - # 8. Restore database - self._log("Restoring database backup...") + # Verify data is cleared + if CharModel.objects.exists() or TextModel.objects.exists(): + raise AssertionError("Test data was not properly cleared") + if os.path.exists(test_file): + raise AssertionError("Media files were not properly cleared") + self._log("Test data cleared successfully") + + # Run restore + self._log("Running database restore...") execute_from_command_line(["", "dbrestore", "--noinput"]) - # 9. Restore media - self._log("Restoring media backup...") + self._log("Running media restore...") execute_from_command_line(["", "mediarestore", "--noinput"]) - # 10. Verify restored data - self._verify_test_data(expected_char_count, expected_text_count) + # Verify restored data + self._verify_test_data(char_obj, text_obj) - # 11. Verify restored media + # Verify restored media if not os.path.exists(test_file): raise AssertionError(f"Media file not restored: {test_file}") @@ -317,62 +362,110 @@ def run_test(self): if content != "test content": raise AssertionError(f"Media file content mismatch: expected 'test content', got '{content}'") - self._log(f"MySQL live test for {self.connector_name} completed successfully") + self._log(f"{SYMBOL_PASS} {self.connector_name} backup/restore test PASSED") return True except RuntimeError as e: - if "Could not connect to MySQL" in str(e): + if "Could not connect to MySQL" in str(e) or "MySQL client tools" in str(e): self._log(f"MySQL not available, skipping test: {e}") return "skipped" else: - self._log(f"MySQL live test for {self.connector_name} failed: {e}") + self._log(f"{SYMBOL_FAIL} {self.connector_name} backup/restore test FAILED: {e}") return False except Exception as e: - self._log(f"MySQL live test for {self.connector_name} failed: {e}") + self._log(f"{SYMBOL_FAIL} {self.connector_name} backup/restore test FAILED: {e}") return False + finally: - # Cleanup self.mysql_runner.cleanup() +def _connector_test_entry(connector_name: str, verbose: bool): # pragma: no cover - executed in subprocess + """Subprocess entry point to run a single connector test. + + Needs to be at module top-level so it can be imported/pickled on Windows + (the 'spawn' start method). Exits with status code 1 if the test fails, + 2 if the test is skipped. + """ + test_runner = MySQLLiveTest(connector_name, verbose) + result = test_runner.run_backup_restore_test() + if result == "skipped": + sys.exit(2) # Special exit code for skipped + elif not result: + sys.exit(1) + + +def run_single_connector_test(connector_name, verbose=False): + """Run a test for a single connector in isolation using a subprocess. + + On Windows, multiprocessing with nested (local) functions fails because they + are not picklable under the 'spawn' start method. We therefore provide a + top-level function as the process target. If an unexpected multiprocessing + failure occurs on Windows (e.g., permissions), we gracefully fall back to + in-process execution to avoid masking all connector results. + """ + + # Normal path: run in a separate process for isolation + def _run_subprocess(): # local helper kept simple; not used as target + process_local = Process(target=_connector_test_entry, args=(connector_name, verbose)) + process_local.start() + process_local.join() + return process_local + + try: + process = _run_subprocess() + if process.exitcode is None: + return False + if process.exitcode == 2: + return "skipped" + if process.exitcode != 0 and os.name == "nt": # Fallback path on Windows + # Retry in-process so at least we capture a meaningful failure message + test_runner = MySQLLiveTest(connector_name, verbose) + return test_runner.run_backup_restore_test() + return process.exitcode == 0 + except AttributeError as exc: # Defensive: pickling or spawn related + if os.name == "nt": + print(f"{SYMBOL_FAIL} Multiprocessing issue on Windows ({exc}); running in-process instead.") + test_runner = MySQLLiveTest(connector_name, verbose) + return test_runner.run_backup_restore_test() + raise + + def _run_all(connectors, verbose: bool) -> int: """Run tests for all connectors.""" overall_success = True results = {} skipped_count = 0 - for name in connectors: - cmd = [sys.executable, __file__, "--connector", name] - if verbose: - cmd.append("-v") + for connector in connectors: + print(f"\n{SYMBOL_TEST} Testing {connector}...") + result = run_single_connector_test(connector, verbose=verbose) - print(f"\n{SYMBOL_TEST} Testing {name}...") - proc = subprocess.run(cmd, check=False) - - if proc.returncode == 0: - passed = True - status = f"{SYMBOL_PASS} PASSED" - elif proc.returncode == 2: # Special exit code for skipped + if result == "skipped": passed = "skipped" status = f"⏭️ SKIPPED" skipped_count += 1 + elif result: + passed = True + status = f"{SYMBOL_PASS} PASSED" else: passed = False status = f"{SYMBOL_FAIL} FAILED" overall_success = False - results[name] = passed - print(f" {name}: {status}") - + results[connector] = passed + print(f" {connector}: {status}") + + # Summary print(f"\n{SYMBOL_SUMMARY} MySQL Connector Test Summary") - for name, passed in results.items(): + for connector, passed in results.items(): if passed == "skipped": status = "⏭️ " elif passed: status = SYMBOL_PASS else: status = SYMBOL_FAIL - print(f" {status} {name}") + print(f" {status} {connector}") if skipped_count == len(connectors): print(" ℹ️ All tests skipped (MySQL not available)") @@ -381,29 +474,31 @@ def _run_all(connectors, verbose: bool) -> int: return 0 if overall_success else 1 -def main() -> int: - """Main entry point for MySQL live tests.""" +def main(): + """Main entry point for the script.""" import argparse - parser = argparse.ArgumentParser(description="Run live MySQL functional tests for django-dbbackup") - parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output") + parser = argparse.ArgumentParser(description="Run live MySQL tests for django-dbbackup") parser.add_argument( "--connector", - choices=MYSQL_CONNECTORS, default="MysqlDumpConnector", + choices=MYSQL_CONNECTORS, help="MySQL connector to test (default: %(default)s)", ) + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output") parser.add_argument("--all", action="store_true", help="Test all MySQL connectors") - + args = parser.parse_args() - verbose = args.verbose + connectors_to_test = MYSQL_CONNECTORS if args.all else [args.connector] + + print(f"{SYMBOL_MYSQL} Starting MySQL Live Tests for django-dbbackup (Isolated)") + if args.all: - return _run_all(MYSQL_CONNECTORS, verbose) - + return _run_all(MYSQL_CONNECTORS, args.verbose) + # Run single connector test - test = MySQLLiveTest(args.connector, verbose=verbose) - result = test.run_test() + result = run_single_connector_test(args.connector, verbose=args.verbose) if result == "skipped": return 2 # Special exit code for skipped From acfa6d4d204708640ed1cd962b1f5be4fd207fb1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 21 Aug 2025 23:36:06 -0700 Subject: [PATCH 05/19] fix mysql setup workflow --- .github/workflows/ci.yml | 4 +++ .github/workflows/copilot-setup-steps.yml | 36 +++++++++++++++++++++++ docs/src/contributing.md | 2 +- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/copilot-setup-steps.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 904c97b9..51637470 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,10 @@ jobs: mysql-root-password: "root" - name: Run functional tests run: hatch run functional:all -v + - name: Install MySQL + uses: kayqueGovetri/setup-mysql@v0.0.3 + with: + root-password: "mysql" build-python: name: Build Python diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 00000000..b550ef7e --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,36 @@ +name: "Copilot Setup Steps" + +# Automatically run the setup steps when they are changed to allow for easy validation, and +# allow manual testing through the repository's "Actions" tab +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. + copilot-setup-steps: + runs-on: ubuntu-latest + + # Set the permissions to the lowest permissions possible needed for your steps. + # Copilot will be given its own token for its operations. + permissions: + # If you want to clone the repository as part of your setup steps, for example to + # install dependencies, you'll need the `contents: read` permission. If you don't + # clone the repository in your setup steps, Copilot will do this for you automatically + # after the steps complete. + contents: read + + # You can define any steps you want, and they will run before the agent starts. + # If you do not check out your code, Copilot will do this for you. + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install MySQL + uses: kayqueGovetri/setup-mysql + with: + root-password: "mysql" diff --git a/docs/src/contributing.md b/docs/src/contributing.md index 60279011..9ff862bc 100644 --- a/docs/src/contributing.md +++ b/docs/src/contributing.md @@ -115,7 +115,7 @@ environment variables adjust its behavior: Database engine to use. See `django.db.backends` for default backends. -**`DB_NAME`** - Default: `:memory:` +**`DB_NAME`** - Default: `/tmp/test_db.sqlite3` Database name. Adjust for non-SQLite backends. From f09f31bdefa0e28f4dcba4c9170f63ec3d47b969 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 21 Aug 2025 23:40:25 -0700 Subject: [PATCH 06/19] fix failed merge --- .github/workflows/ci.yml | 9 ++------- .github/workflows/copilot-setup-steps.yml | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51637470..f6fa8e54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,16 +116,11 @@ jobs: PGSERVICE: postgres - name: Setup MySQL if: matrix.os == 'ubuntu-latest' - uses: kayqueGovetri/setup-mysql@v3.0.0 + uses: kayqueGovetri/setup-mysql@@v0.0.3 with: - mysql-version: "8.0" - mysql-root-password: "root" + root-password: "mysql" - name: Run functional tests run: hatch run functional:all -v - - name: Install MySQL - uses: kayqueGovetri/setup-mysql@v0.0.3 - with: - root-password: "mysql" build-python: name: Build Python diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index b550ef7e..5e529c25 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -31,6 +31,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Install MySQL - uses: kayqueGovetri/setup-mysql + uses: kayqueGovetri/setup-mysql@v0.0.3 with: root-password: "mysql" From 688d794e653ff19931ab1394fd2d6429bf81c59b Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 21 Aug 2025 23:41:39 -0700 Subject: [PATCH 07/19] fix incorrect numbering in instructions --- .github/copilot-instructions.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 489ff188..166e06bb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -62,12 +62,12 @@ Always test backup and restore functionality after making changes using function 2. **Individual Backend Tests:** ```bash - hatch run functional:sqlite --all # SQLite connectors - hatch run functional:postgres --all # PostgreSQL connectors - hatch run functional:mysql --all # MySQL connectors + hatch run functional:sqlite --all -v # SQLite connectors + hatch run functional:postgres --all -v # PostgreSQL connectors + hatch run functional:mysql --all -v # MySQL connectors ``` -2. **Manual Database Test (if needed):** +3. **Manual Database Test (if needed):** ```bash hatch shell functional @@ -77,7 +77,7 @@ Always test backup and restore functionality after making changes using function python -m django dbrestore --noinput ``` -3. **Manual Media Test (if needed):** +4. **Manual Media Test (if needed):** ```bash hatch shell functional @@ -89,7 +89,7 @@ Always test backup and restore functionality after making changes using function ls tmp/media/ # should show restored test.txt ``` -4. **Single Backend Functional Runs (if triaging):** +5. **Single Backend Functional Runs (if triaging):** ```bash hatch run functional:sqlite --all hatch run functional:postgres --all @@ -163,7 +163,7 @@ hatch test # Run all tests across matrix hatch test --python 3.12 # Test specific Python version subset hatch run functional:all # Functional tests (SQLite + PostgreSQL) hatch run functional:sqlite --all # Functional tests (SQLite only) -hatch run functional:postgres --all # Functional tests (PostgreSQL only) +hatch run functional:postgres --all # Functional tests (PostgreSQL only) hatch run functional:mysql --all # Functional tests (MySQL only) hatch run lint:check # Lint (ruff + pylint) hatch run lint:format # Auto-format From a4a7157f403a472ae356644a5c1aa01c9cc77a79 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 21 Aug 2025 23:43:50 -0700 Subject: [PATCH 08/19] fix typos --- .github/copilot-instructions.md | 14 +++----------- .github/workflows/ci.yml | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 166e06bb..f83078eb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -59,15 +59,7 @@ Always test backup and restore functionality after making changes using function hatch run functional:all ``` -2. **Individual Backend Tests:** - - ```bash - hatch run functional:sqlite --all -v # SQLite connectors - hatch run functional:postgres --all -v # PostgreSQL connectors - hatch run functional:mysql --all -v # MySQL connectors - ``` - -3. **Manual Database Test (if needed):** +2. **Manual Database Test (if needed):** ```bash hatch shell functional @@ -77,7 +69,7 @@ Always test backup and restore functionality after making changes using function python -m django dbrestore --noinput ``` -4. **Manual Media Test (if needed):** +3. **Manual Media Test (if needed):** ```bash hatch shell functional @@ -89,7 +81,7 @@ Always test backup and restore functionality after making changes using function ls tmp/media/ # should show restored test.txt ``` -5. **Single Backend Functional Runs (if triaging):** +4. **Single Backend Functional Runs:** ```bash hatch run functional:sqlite --all hatch run functional:postgres --all diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6fa8e54..8743cd07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: PGSERVICE: postgres - name: Setup MySQL if: matrix.os == 'ubuntu-latest' - uses: kayqueGovetri/setup-mysql@@v0.0.3 + uses: kayqueGovetri/setup-mysql@v0.0.3 with: root-password: "mysql" - name: Run functional tests From 9ae2da7bf17ae7c653c403d4933fb02aacbf83d3 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 00:04:20 -0700 Subject: [PATCH 09/19] fix arg name --- .github/workflows/ci.yml | 2 +- .github/workflows/copilot-setup-steps.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8743cd07..430e7556 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: if: matrix.os == 'ubuntu-latest' uses: kayqueGovetri/setup-mysql@v0.0.3 with: - root-password: "mysql" + mysq_root_password: "mysql" - name: Run functional tests run: hatch run functional:all -v diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5e529c25..028f129a 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -33,4 +33,4 @@ jobs: - name: Install MySQL uses: kayqueGovetri/setup-mysql@v0.0.3 with: - root-password: "mysql" + mysq_root_password: "mysql" From 902452ad6dd75e82d8247ae707419f1fd3117628 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 00:05:29 -0700 Subject: [PATCH 10/19] fix typo --- .github/workflows/ci.yml | 2 +- .github/workflows/copilot-setup-steps.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 430e7556..6b8c9bc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: if: matrix.os == 'ubuntu-latest' uses: kayqueGovetri/setup-mysql@v0.0.3 with: - mysq_root_password: "mysql" + mysql_root_password: "mysql" - name: Run functional tests run: hatch run functional:all -v diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 028f129a..b23b07d8 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -33,4 +33,4 @@ jobs: - name: Install MySQL uses: kayqueGovetri/setup-mysql@v0.0.3 with: - mysq_root_password: "mysql" + mysql_root_password: "mysql" From 3198465656c28b3da0c91d9cb7608c9e61e8bdf3 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 19:46:52 -0700 Subject: [PATCH 11/19] Bump workflow --- .github/workflows/ci.yml | 2 +- .github/workflows/copilot-setup-steps.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b8c9bc2..6e54bd4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: PGSERVICE: postgres - name: Setup MySQL if: matrix.os == 'ubuntu-latest' - uses: kayqueGovetri/setup-mysql@v0.0.3 + uses: kayqueGovetri/setup-mysql@v0.0.4 with: mysql_root_password: "mysql" - name: Run functional tests diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index b23b07d8..9bc77de2 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -31,6 +31,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Install MySQL - uses: kayqueGovetri/setup-mysql@v0.0.3 + uses: kayqueGovetri/setup-mysql@v0.0.4 with: mysql_root_password: "mysql" From 968b6ffb794a5267257e8158f2d08e1d8160ad5f Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 19:52:18 -0700 Subject: [PATCH 12/19] Fix MySQL root password setting --- scripts/mysql_live_test.py | 151 +++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 75 deletions(-) diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index 4d21c4b3..672813a0 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -6,7 +6,7 @@ python scripts/mysql_live_test.py --all It provides end-to-end validation of MySQL backup/restore functionality using the -available connectors and mirrors the visual layout & summary style of the SQLite and +available connectors and mirrors the visual layout & summary style of the SQLite and PostgreSQL live tests for consistency. Exit code 0 on success (all tested connectors passed), 1 on failure. @@ -46,28 +46,28 @@ class MySQLTestRunner: """Manages a test database on the existing MySQL instance.""" - + def __init__(self, verbose=False): self.verbose = verbose self.temp_dir = None self.test_db_name = f"dbbackup_test_{int(time.time())}" self.user = "dbbackup_test_user" - self.password = "test_password_123" + self.password = "mysql" self.host = "localhost" self.port = 3306 self.superuser = "root" self.db_created = False self.user_created = False - + def _log(self, message): """Log a message if verbose mode is enabled.""" if self.verbose: print(f"[MySQL Test] {message}") - + def _run_command(self, cmd, check=True, capture_output=False, **kwargs): """Run a command and optionally check for errors.""" self._log(f"Running: {' '.join(cmd)}") - + result = subprocess.run(cmd, capture_output=capture_output, text=True, **kwargs) if check and result.returncode != 0: error_msg = f"Command failed with exit code {result.returncode}: {' '.join(cmd)}" @@ -78,10 +78,10 @@ def _run_command(self, cmd, check=True, capture_output=False, **kwargs): error_msg += f"\nSTDERR: {result.stderr}" raise RuntimeError(error_msg) return result - + def setup_mysql(self): """Set up a test database on the existing MySQL instance.""" - + if not shutil.which("mysql") or not shutil.which("mysqldump"): install_instructions = "" if os.name == "posix": @@ -105,14 +105,14 @@ def setup_mysql(self): except Exception as e: self.cleanup() raise RuntimeError(f"Failed to set up MySQL: {e}") from e - + def _create_test_database(self): """Create the test database.""" self._log(f"Creating test database: {self.test_db_name}") - + # Try different MySQL authentication methods mysql_commands = [] - + if GITHUB_ACTIONS: self._log("GitHub Actions detected, using setup-mysql action configuration") # GitHub Actions with setup-mysql action uses root user with password 'root' @@ -131,34 +131,35 @@ def _create_test_database(self): ] self.user = self.superuser # Use root for simplicity self.password = "" - + # Try to connect and create database create_db_sql = f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};" - + for mysql_cmd in mysql_commands: try: cmd = mysql_cmd + ["-e", create_db_sql] self._log(f"Trying MySQL connection with: {' '.join(mysql_cmd[:3])}") # Don't log password self._run_command(cmd, capture_output=True) self._log("Successfully connected to MySQL and created database") - + # Test the connection works test_cmd = mysql_cmd + ["-e", "SELECT 1;"] self._run_command(test_cmd, capture_output=True) - + # Store the working command for later use self.mysql_base_cmd = mysql_cmd self.db_created = True return - + except RuntimeError as e: self._log(f"MySQL connection failed with {' '.join(mysql_cmd[:3])}: {e}") continue - + # If all methods fail, raise an error - raise RuntimeError("Could not connect to MySQL with any authentication method. " - "Please ensure MySQL is running and accessible.") - + raise RuntimeError( + "Could not connect to MySQL with any authentication method. Please ensure MySQL is running and accessible." + ) + def get_database_config(self): """Get Django database configuration for the test MySQL instance.""" return { @@ -172,53 +173,53 @@ def get_database_config(self): "init_command": "SET sql_mode='STRICT_TRANS_TABLES'", }, } - + def cleanup(self): """Clean up the test database.""" self._log("Cleaning up test database...") - if self.db_created and hasattr(self, 'mysql_base_cmd'): + if self.db_created and hasattr(self, "mysql_base_cmd"): try: self._log(f"Dropping test database: {self.test_db_name}") cmd = self.mysql_base_cmd + ["-e", f"DROP DATABASE IF EXISTS {self.test_db_name};"] self._run_command(cmd, check=False) except Exception as e: self._log(f"Warning: Failed to drop test database: {e}") - - if self.user_created and hasattr(self, 'mysql_base_cmd') and not GITHUB_ACTIONS: + + if self.user_created and hasattr(self, "mysql_base_cmd") and not GITHUB_ACTIONS: try: self._log(f"Dropping test user: {self.user}") cmd = self.mysql_base_cmd + ["-e", f"DROP USER IF EXISTS '{self.user}'@'localhost';"] self._run_command(cmd, check=False) except Exception as e: self._log(f"Warning: Failed to drop test user: {e}") - + if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) class MySQLLiveTest: """Runs live tests against MySQL connectors.""" - + def __init__(self, connector_name="MysqlDumpConnector", verbose=False): self.connector_name = connector_name self.verbose = verbose self.mysql_runner = MySQLTestRunner(verbose=verbose) - + def _log(self, message): """Log a message if verbose mode is enabled.""" if self.verbose: print(f"[Live Test] {message}") - + def _configure_django(self): """Configure Django with the test MySQL database.""" # Configure Django settings os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") - + # Override database settings db_config = self.mysql_runner.get_database_config() os.environ.update({ - "DB_ENGINE": db_config["ENGINE"], + "DB_ENGINE": db_config["ENGINE"], "DB_NAME": db_config["NAME"], "DB_USER": db_config["USER"], "DB_HOST": db_config["HOST"], @@ -228,10 +229,10 @@ def _configure_django(self): os.environ["DB_PASSWORD"] = db_config["PASSWORD"] # Set port as string os.environ["DB_PORT"] = str(db_config["PORT"]) - + # Set connector os.environ["CONNECTOR"] = f"dbbackup.db.mysql.{self.connector_name}" - + # Configure storage for backups - use unique directory per test backup_dir = os.path.join(str(self.mysql_runner.temp_dir), "backups") os.makedirs(backup_dir, exist_ok=True) @@ -241,130 +242,130 @@ def _configure_django(self): "STORAGE_OPTIONS": f"location={backup_dir}", "MEDIA_ROOT": os.path.join(str(self.mysql_runner.temp_dir), "media"), }) - + # Setup Django only if not already configured if not django.apps.apps.ready: django.setup() - + def _create_test_data(self): """Create test data in the database.""" self._log("Creating test data...") - + # Run migrations execute_from_command_line(["", "migrate", "--noinput"]) - + # Create test models from tests.testapp.models import CharModel, TextModel - + # Create some test data (CharModel has max_length=10) char_obj = CharModel.objects.create(field="test_char") # 9 chars, fits in 10 text_obj = TextModel.objects.create(field="test text content for backup") - + self._log(f"Created CharModel: {char_obj}") self._log(f"Created TextModel: {text_obj}") - + return char_obj, text_obj - + def _verify_test_data(self, expected_char_obj, expected_text_obj): """Verify that test data exists and matches expectations.""" from tests.testapp.models import CharModel, TextModel - + char_objs = CharModel.objects.all() text_objs = TextModel.objects.all() - + self._log(f"Found {char_objs.count()} CharModel objects") self._log(f"Found {text_objs.count()} TextModel objects") - + if char_objs.count() != 1 or text_objs.count() != 1: raise AssertionError( f"Expected 1 of each model, found {char_objs.count()} CharModel and {text_objs.count()} TextModel" ) - + char_obj = char_objs.first() text_obj = text_objs.first() - + if char_obj.field != expected_char_obj.field: raise AssertionError( f"CharModel field mismatch: expected '{expected_char_obj.field}', got '{char_obj.field}'" ) - + if text_obj.field != expected_text_obj.field: raise AssertionError( f"TextModel field mismatch: expected '{expected_text_obj.field}', got '{text_obj.field}'" ) - + self._log("Test data verification passed") def run_backup_restore_test(self): """Run a complete backup and restore test cycle.""" self._log(f"Starting backup/restore test with {self.connector_name}") - + try: # Setup MySQL self.mysql_runner.setup_mysql() - + # Configure Django self._configure_django() - + # Create test data char_obj, text_obj = self._create_test_data() - + # Run backup self._log("Running database backup...") execute_from_command_line(["", "dbbackup", "--noinput"]) - + # Create media files and backup media_dir = os.environ["MEDIA_ROOT"] os.makedirs(media_dir, exist_ok=True) - + # Create test media files test_file = os.path.join(media_dir, "test.txt") with open(test_file, "w") as f: f.write("test content") - + self._log("Running media backup...") execute_from_command_line(["", "mediabackup", "--noinput"]) - + # Clear test data self._log("Clearing test data...") from tests.testapp.models import CharModel, TextModel - + CharModel.objects.all().delete() TextModel.objects.all().delete() - + # Remove media files if os.path.exists(test_file): os.remove(test_file) - + # Verify data is cleared if CharModel.objects.exists() or TextModel.objects.exists(): raise AssertionError("Test data was not properly cleared") if os.path.exists(test_file): raise AssertionError("Media files were not properly cleared") self._log("Test data cleared successfully") - + # Run restore self._log("Running database restore...") execute_from_command_line(["", "dbrestore", "--noinput"]) - + self._log("Running media restore...") execute_from_command_line(["", "mediarestore", "--noinput"]) - + # Verify restored data self._verify_test_data(char_obj, text_obj) - + # Verify restored media if not os.path.exists(test_file): raise AssertionError(f"Media file not restored: {test_file}") - + with open(test_file, "r") as f: content = f.read() if content != "test content": raise AssertionError(f"Media file content mismatch: expected 'test content', got '{content}'") - + self._log(f"{SYMBOL_PASS} {self.connector_name} backup/restore test PASSED") return True - + except RuntimeError as e: if "Could not connect to MySQL" in str(e) or "MySQL client tools" in str(e): self._log(f"MySQL not available, skipping test: {e}") @@ -375,7 +376,7 @@ def run_backup_restore_test(self): except Exception as e: self._log(f"{SYMBOL_FAIL} {self.connector_name} backup/restore test FAILED: {e}") return False - + finally: self.mysql_runner.cleanup() @@ -436,11 +437,11 @@ def _run_all(connectors, verbose: bool) -> int: overall_success = True results = {} skipped_count = 0 - + for connector in connectors: print(f"\n{SYMBOL_TEST} Testing {connector}...") result = run_single_connector_test(connector, verbose=verbose) - + if result == "skipped": passed = "skipped" status = f"⏭️ SKIPPED" @@ -452,7 +453,7 @@ def _run_all(connectors, verbose: bool) -> int: passed = False status = f"{SYMBOL_FAIL} FAILED" overall_success = False - + results[connector] = passed print(f" {connector}: {status}") @@ -466,18 +467,18 @@ def _run_all(connectors, verbose: bool) -> int: else: status = SYMBOL_FAIL print(f" {status} {connector}") - + if skipped_count == len(connectors): print(" ℹ️ All tests skipped (MySQL not available)") return 0 # Don't fail if MySQL is not available - + return 0 if overall_success else 1 def main(): """Main entry point for the script.""" import argparse - + parser = argparse.ArgumentParser(description="Run live MySQL tests for django-dbbackup") parser.add_argument( "--connector", @@ -489,7 +490,7 @@ def main(): parser.add_argument("--all", action="store_true", help="Test all MySQL connectors") args = parser.parse_args() - + connectors_to_test = MYSQL_CONNECTORS if args.all else [args.connector] print(f"{SYMBOL_MYSQL} Starting MySQL Live Tests for django-dbbackup (Isolated)") @@ -499,7 +500,7 @@ def main(): # Run single connector test result = run_single_connector_test(args.connector, verbose=args.verbose) - + if result == "skipped": return 2 # Special exit code for skipped elif result: @@ -509,4 +510,4 @@ def main(): if __name__ == "__main__": # pragma: no cover - executed as script - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From da605b0a4025f564c639fb15261e198a279ba73f Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:16:18 -0700 Subject: [PATCH 13/19] Fix mysql user generation --- scripts/mysql_live_test.py | 78 +++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index 672813a0..12893a6f 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -110,50 +110,40 @@ def _create_test_database(self): """Create the test database.""" self._log(f"Creating test database: {self.test_db_name}") - # Try different MySQL authentication methods - mysql_commands = [] - - if GITHUB_ACTIONS: - self._log("GitHub Actions detected, using setup-mysql action configuration") - # GitHub Actions with setup-mysql action uses root user with password 'root' - mysql_commands = [ - ["mysql", "-u", "root", "-proot", "-h", "localhost"], - ["mysql", "-u", "root", "-proot"], - ] - self.user = self.superuser - self.password = "root" # setup-mysql action default - else: - # For local development, try various authentication methods - mysql_commands = [ - ["sudo", "mysql"], # auth_socket (common on Ubuntu) - ["mysql", "-u", "root"], # no password - ["mysql", "-u", "root", "-p"], # password prompt (will fail in automation) - ] - self.user = self.superuser # Use root for simplicity - self.password = "" - - # Try to connect and create database - create_db_sql = f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};" - - for mysql_cmd in mysql_commands: - try: - cmd = mysql_cmd + ["-e", create_db_sql] - self._log(f"Trying MySQL connection with: {' '.join(mysql_cmd[:3])}") # Don't log password - self._run_command(cmd, capture_output=True) - self._log("Successfully connected to MySQL and created database") - - # Test the connection works - test_cmd = mysql_cmd + ["-e", "SELECT 1;"] - self._run_command(test_cmd, capture_output=True) - - # Store the working command for later use - self.mysql_base_cmd = mysql_cmd - self.db_created = True - return - - except RuntimeError as e: - self._log(f"MySQL connection failed with {' '.join(mysql_cmd[:3])}: {e}") - continue + mysql_admin_cmd = ["mysql", "-u", "root", f"--password={self.password}"] + + try: + # Test the connection works + test_cmd = mysql_admin_cmd + ["-e", "SELECT 1;"] + self._run_command(test_cmd, capture_output=True) + + # Create the test database + cmd = mysql_admin_cmd + ["-e", f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};"] + self._log(f"Trying MySQL connection with: {' '.join(mysql_admin_cmd[:3])}") # Don't log password + self._run_command(cmd, capture_output=True) + self._log("Successfully connected to MySQL and created database") + + # Create a dedicated user for the test database + self._run_command( + mysql_admin_cmd + + [ + "-e", + ( + f"CREATE USER IF NOT EXISTS '{self.user}'@'localhost' IDENTIFIED BY '{self.password}';" + f"GRANT ALL PRIVILEGES ON {self.test_db_name}.* TO '{self.user}'@'localhost';" + "FLUSH PRIVILEGES;" + ), + ], + capture_output=True, + ) + + # Store the working command for later use + self.mysql_base_cmd = ["mysql", "-u", self.user, f"--password={self.password}"] + self.db_created = True + return + + except RuntimeError as e: + self._log(f"MySQL connection failed with {' '.join(mysql_admin_cmd[:3])}: {e}") # If all methods fail, raise an error raise RuntimeError( From b5729cb91314513664d663ad5c05e995dd736ba3 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:27:00 -0700 Subject: [PATCH 14/19] force mysql connections over tcp --- scripts/mysql_live_test.py | 107 ++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index 12893a6f..66e8660d 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -51,11 +51,23 @@ def __init__(self, verbose=False): self.verbose = verbose self.temp_dir = None self.test_db_name = f"dbbackup_test_{int(time.time())}" - self.user = "dbbackup_test_user" - self.password = "mysql" - self.host = "localhost" - self.port = 3306 - self.superuser = "root" + # Allow environment overrides so CI or developers can point at + # an existing server without editing the script. These defaults + # mirror the prior hard‑coded values but now support more setups + # (e.g. root account with socket auth, custom host, empty password). + self.user = os.getenv("MYSQL_TEST_USER", "dbbackup_test_user") + # Password used for the dedicated test user we create + self.password = os.getenv("MYSQL_TEST_PASSWORD", "mysql") + # Force TCP by default (127.0.0.1) to avoid relying on a Unix domain + # socket path that varies across distros (/var/run vs /run) and to + # work with Docker/remote services. A user can still supply the + # classic 'localhost' or a remote hostname via env var. + self.host = os.getenv("MYSQL_HOST", "127.0.0.1") + self.port = int(os.getenv("MYSQL_PORT", "3306")) + # Superuser used to bootstrap the test DB & user. Allow empty password + # (common with auth_socket); we'll attempt multiple auth modes. + self.superuser = os.getenv("MYSQL_SUPERUSER", "root") + self.superpassword = os.getenv("MYSQL_SUPERPASSWORD", os.getenv("MYSQL_ROOT_PASSWORD", "mysql")) self.db_created = False self.user_created = False @@ -110,45 +122,49 @@ def _create_test_database(self): """Create the test database.""" self._log(f"Creating test database: {self.test_db_name}") - mysql_admin_cmd = ["mysql", "-u", "root", f"--password={self.password}"] + # Build two candidate admin commands: with password (if provided) & without. + base_common = ["-h", self.host, "-P", str(self.port), "--protocol=TCP"] + candidate_cmds = [] + if self.superpassword: + candidate_cmds.append(["mysql", "-u", self.superuser, f"--password={self.superpassword}"] + base_common) + candidate_cmds.append(["mysql", "-u", self.superuser] + base_common) - try: - # Test the connection works - test_cmd = mysql_admin_cmd + ["-e", "SELECT 1;"] - self._run_command(test_cmd, capture_output=True) - - # Create the test database - cmd = mysql_admin_cmd + ["-e", f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};"] - self._log(f"Trying MySQL connection with: {' '.join(mysql_admin_cmd[:3])}") # Don't log password - self._run_command(cmd, capture_output=True) - self._log("Successfully connected to MySQL and created database") - - # Create a dedicated user for the test database - self._run_command( - mysql_admin_cmd - + [ - "-e", - ( - f"CREATE USER IF NOT EXISTS '{self.user}'@'localhost' IDENTIFIED BY '{self.password}';" - f"GRANT ALL PRIVILEGES ON {self.test_db_name}.* TO '{self.user}'@'localhost';" - "FLUSH PRIVILEGES;" - ), - ], - capture_output=True, - ) - - # Store the working command for later use - self.mysql_base_cmd = ["mysql", "-u", self.user, f"--password={self.password}"] - self.db_created = True - return - - except RuntimeError as e: - self._log(f"MySQL connection failed with {' '.join(mysql_admin_cmd[:3])}: {e}") + last_err = None + for cmd_parts in candidate_cmds: + try: + self._log( + "Trying MySQL connection with: " + + " ".join([c for c in cmd_parts if not c.startswith("--password=")]) + ) + # Smoke test + self._run_command(cmd_parts + ["-e", "SELECT 1;"], capture_output=True) + # Create database + self._run_command( + cmd_parts + ["-e", f"CREATE DATABASE IF NOT EXISTS {self.test_db_name};"], capture_output=True + ) + # Create user (both '%' and 'localhost' host patterns) + pw_clause = f" IDENTIFIED BY '{self.password}'" if self.password else "" + create_user_sql = ( + f"CREATE USER IF NOT EXISTS '{self.user}'@'%' {pw_clause};" + f"CREATE USER IF NOT EXISTS '{self.user}'@'localhost' {pw_clause};" + f"GRANT ALL PRIVILEGES ON {self.test_db_name}.* TO '{self.user}'@'%';" + f"GRANT ALL PRIVILEGES ON {self.test_db_name}.* TO '{self.user}'@'localhost';" + "FLUSH PRIVILEGES;" + ) + self._run_command(cmd_parts + ["-e", create_user_sql], capture_output=True, check=False) + self.user_created = True + self.mysql_base_cmd = cmd_parts # keep for cleanup + self.db_created = True + self._log("Successfully connected to MySQL and created database & user over TCP") + return + except RuntimeError as exc: + last_err = exc + self._log(f"Attempt failed: {exc}") - # If all methods fail, raise an error raise RuntimeError( - "Could not connect to MySQL with any authentication method. Please ensure MySQL is running and accessible." - ) + "Could not connect to MySQL with any authentication method. Ensure the server is running " + "and environment vars MYSQL_HOST / MYSQL_PORT / MYSQL_SUPERUSER / MYSQL_SUPERPASSWORD are correct." + ) from last_err def get_database_config(self): """Get Django database configuration for the test MySQL instance.""" @@ -157,6 +173,7 @@ def get_database_config(self): "NAME": self.test_db_name, "USER": self.user, "PASSWORD": self.password, + # Use provided host explicitly (avoid default socket resolution) "HOST": self.host, "PORT": self.port, "OPTIONS": { @@ -179,7 +196,13 @@ def cleanup(self): if self.user_created and hasattr(self, "mysql_base_cmd") and not GITHUB_ACTIONS: try: self._log(f"Dropping test user: {self.user}") - cmd = self.mysql_base_cmd + ["-e", f"DROP USER IF EXISTS '{self.user}'@'localhost';"] + # Drop both '%' and 'localhost' host specs just in case + drop_stmt = ( + f"DROP USER IF EXISTS '{self.user}'@'%';" + f"DROP USER IF EXISTS '{self.user}'@'localhost';" + "FLUSH PRIVILEGES;" + ) + cmd = self.mysql_base_cmd + ["-e", drop_stmt] self._run_command(cmd, check=False) except Exception as e: self._log(f"Warning: Failed to drop test user: {e}") From ef2eefee4c90aadb9f016d7586980692c417aab1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:36:19 -0700 Subject: [PATCH 15/19] Fix windows ascii errors --- scripts/_utils.py | 4 ++++ scripts/mysql_live_test.py | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/_utils.py b/scripts/_utils.py index 1b4e37ef..c6270b4b 100644 --- a/scripts/_utils.py +++ b/scripts/_utils.py @@ -27,6 +27,8 @@ "FAIL": "❌", "SUMMARY": "πŸ“Š", "TEST": "πŸ“‹", + "SKIP": "⏭️", + "INFO": "ℹ️", "PG": "🐘", "MYSQL": "🐬", } @@ -36,6 +38,8 @@ "FAIL": "FAIL:", "SUMMARY": "SUMMARY:", "TEST": "TEST:", + "SKIP": "SKIP:", + "INFO": "INFO:", "PG": ">>", "MYSQL": ">>", } diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index 66e8660d..ca191b13 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -29,6 +29,8 @@ SYMBOL_SUMMARY = _SYMS["SUMMARY"] SYMBOL_MYSQL = _SYMS["MYSQL"] SYMBOL_TEST = _SYMS["TEST"] +SYMBOL_SKIP = _SYMS["SKIP"] +SYMBOL_INFO = _SYMS["INFO"] # Available MySQL connectors MYSQL_CONNECTORS = [ @@ -457,7 +459,7 @@ def _run_all(connectors, verbose: bool) -> int: if result == "skipped": passed = "skipped" - status = f"⏭️ SKIPPED" + status = f"{SYMBOL_SKIP} SKIPPED" skipped_count += 1 elif result: passed = True @@ -474,7 +476,7 @@ def _run_all(connectors, verbose: bool) -> int: print(f"\n{SYMBOL_SUMMARY} MySQL Connector Test Summary") for connector, passed in results.items(): if passed == "skipped": - status = "⏭️ " + status = SYMBOL_SKIP elif passed: status = SYMBOL_PASS else: @@ -482,7 +484,7 @@ def _run_all(connectors, verbose: bool) -> int: print(f" {status} {connector}") if skipped_count == len(connectors): - print(" ℹ️ All tests skipped (MySQL not available)") + print(f" {SYMBOL_INFO} All tests skipped (MySQL not available)") return 0 # Don't fail if MySQL is not available return 0 if overall_success else 1 @@ -504,8 +506,6 @@ def main(): args = parser.parse_args() - connectors_to_test = MYSQL_CONNECTORS if args.all else [args.connector] - print(f"{SYMBOL_MYSQL} Starting MySQL Live Tests for django-dbbackup (Isolated)") if args.all: From f090104e21f9111882c2078b9c3ea1e6a90dc2e6 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:37:54 -0700 Subject: [PATCH 16/19] Try localhost instead of local addr --- scripts/mysql_live_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index ca191b13..dcf47e9d 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -64,7 +64,7 @@ def __init__(self, verbose=False): # socket path that varies across distros (/var/run vs /run) and to # work with Docker/remote services. A user can still supply the # classic 'localhost' or a remote hostname via env var. - self.host = os.getenv("MYSQL_HOST", "127.0.0.1") + self.host = os.getenv("MYSQL_HOST", "localhost") self.port = int(os.getenv("MYSQL_PORT", "3306")) # Superuser used to bootstrap the test DB & user. Allow empty password # (common with auth_socket); we'll attempt multiple auth modes. From 408aab1a7e829783272cdefb3d2b307dc815f763 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:43:08 -0700 Subject: [PATCH 17/19] explicity set port --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e54bd4e..8f0215d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,7 @@ jobs: uses: kayqueGovetri/setup-mysql@v0.0.4 with: mysql_root_password: "mysql" + mysql_port: 3306 - name: Run functional tests run: hatch run functional:all -v From 7b8cb778d1751ec41cfcc82ae43e2b00652985cb Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:46:20 -0700 Subject: [PATCH 18/19] setup mysql earlier? --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f0215d7..757a90f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,12 @@ jobs: cache: pip - name: Install dependencies run: python -m pip install --upgrade pip hatch uv + - name: Setup MySQL + if: matrix.os == 'ubuntu-latest' + uses: kayqueGovetri/setup-mysql@v0.0.4 + with: + mysql_root_password: "mysql" + mysql_port: 3306 - name: Setup postgres uses: ikalnytskyi/action-setup-postgres@v7 - run: psql postgresql://postgres:postgres@localhost:5432/postgres -c "SELECT 1" @@ -114,12 +120,6 @@ jobs: - run: psql -c "SELECT 1" env: PGSERVICE: postgres - - name: Setup MySQL - if: matrix.os == 'ubuntu-latest' - uses: kayqueGovetri/setup-mysql@v0.0.4 - with: - mysql_root_password: "mysql" - mysql_port: 3306 - name: Run functional tests run: hatch run functional:all -v From 4d48990ee335ae71b0b12d1ee6802a80787a499e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 22 Aug 2025 20:55:10 -0700 Subject: [PATCH 19/19] Allow github actions to fail if mysql is missing --- scripts/mysql_live_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_live_test.py b/scripts/mysql_live_test.py index dcf47e9d..9f0163ff 100644 --- a/scripts/mysql_live_test.py +++ b/scripts/mysql_live_test.py @@ -384,7 +384,7 @@ def run_backup_restore_test(self): except RuntimeError as e: if "Could not connect to MySQL" in str(e) or "MySQL client tools" in str(e): self._log(f"MySQL not available, skipping test: {e}") - return "skipped" + return False if GITHUB_ACTIONS else "skipped" else: self._log(f"{SYMBOL_FAIL} {self.connector_name} backup/restore test FAILED: {e}") return False