From 1a29e8119d0476b46b74dc30d0b31330a3e96979 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Thu, 8 Oct 2020 21:07:02 +0200 Subject: [PATCH 1/5] devops: added Docker image --- .dockerignore | 22 ++++++++++ .github/workflows/test_docker.yml | 46 +++++++++++++++++++ Dockerfile | 73 +++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/test_docker.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..38aa1a761 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +playwright/__pycache__/ +playwright/drivers/driver* +playwright/driver-* +driver/node_modules/ +driver/out/ +driver/package-lock.json +playwright.egg-info/ +build/ +dist/ +**/*.pyc +env/ +htmlcov/ +.coverage +.DS_Store +.vscode/ +.eggs +_repo_version.py +coverage.xml +junit/ +htmldocs/ + +!/dist/*manylinux1_x86_64.whl diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml new file mode 100644 index 000000000..8e9410651 --- /dev/null +++ b/.github/workflows/test_docker.yml @@ -0,0 +1,46 @@ +name: Test Docker +on: + push: + paths: + - '.github/workflows/test_docker.yml' + - 'driver/**' + branches: + - master + pull_request: + paths: + - '.github/workflows/test_docker.yml' + - 'driver/**' + branches: + - master +jobs: + build: + timeout-minutes: 60 + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Node.js + uses: actions/setup-node@v1 + with: + node-version: 12.x + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r local-requirements.txt + pip install -e . + - name: Build driver + run: python build_driver.py + - name: Build package + run: python build_package.py + - name: Install + run: python -m playwright install + - name: Build Docker image + run: docker build -t playwright-python:localbuild . + - name: Test + run: | + CONTAINER_ID="$(docker run --rm -v $(pwd):/root/playwright --name playwright-docker-test -d -t playwright-python:localbuild /bin/bash)" + docker exec --workdir /root/playwright/ "${CONTAINER_ID}" pip install -r local-requirements.txt + docker exec --workdir /root/playwright/ "${CONTAINER_ID}" xvfb-run pytest -vv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..33f2158ea --- /dev/null +++ b/Dockerfile @@ -0,0 +1,73 @@ +FROM ubuntu:focal + +# 1. Install latest Python +RUN apt-get update && apt-get install -y python3 python3-pip && \ + update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +# 2. Install WebKit dependencies +RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + libwoff1 \ + libopus0 \ + libwebp6 \ + libwebpdemux2 \ + libenchant1c2a \ + libgudev-1.0-0 \ + libsecret-1-0 \ + libhyphen0 \ + libgdk-pixbuf2.0-0 \ + libegl1 \ + libnotify4 \ + libxslt1.1 \ + libevent-2.1-7 \ + libgles2 \ + libxcomposite1 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libepoxy0 \ + libgtk-3-0 \ + libharfbuzz-icu0 + +# 3. Install gstreamer and plugins to support video playback in WebKit. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgstreamer-gl1.0-0 \ + libgstreamer-plugins-bad1.0-0 \ + gstreamer1.0-plugins-good \ + gstreamer1.0-libav + +# 4. Install Chromium dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libnss3 \ + libxss1 \ + libasound2 \ + fonts-noto-color-emoji \ + libxtst6 + +# 5. Install Firefox dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libdbus-glib-1-2 \ + libxt6 + +# 6. Install ffmpeg to bring in audio and video codecs necessary for playing videos in Firefox. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg + +# 7. (Optional) Install XVFB if there's a need to run browsers in headful mode +RUN apt-get update && apt-get install -y --no-install-recommends \ + xvfb + +# 8. Feature-parity with node.js base images. +RUN apt-get update && apt-get install -y --no-install-recommends git ssh + +# === BAKE BROWSERS INTO IMAGE === + +# 1. Add tip-of-tree Playwright Python package to install its browsers. +# The package should be built beforehand from tip-of-tree Playwright. +COPY ./dist/playwright*manylinux1*.whl /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl + +# 2. Install playwright and then delete the installation. +# Browsers will remain downloaded in `/root/.cache/ms-playwright`. +RUN mkdir /tmp/pw && cd /tmp/pw && \ + pip install /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl && \ + python -m playwright install && \ + rm -rf /tmp/pw && rm /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl From fbba6daeaef2b7f6924312bba18af3e0ff75f059 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Thu, 8 Oct 2020 23:28:26 +0200 Subject: [PATCH 2/5] fix: review comments --- .dockerignore | 23 +---------------------- Dockerfile | 9 ++++++--- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/.dockerignore b/.dockerignore index 38aa1a761..d77391baa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,22 +1 @@ -playwright/__pycache__/ -playwright/drivers/driver* -playwright/driver-* -driver/node_modules/ -driver/out/ -driver/package-lock.json -playwright.egg-info/ -build/ -dist/ -**/*.pyc -env/ -htmlcov/ -.coverage -.DS_Store -.vscode/ -.eggs -_repo_version.py -coverage.xml -junit/ -htmldocs/ - -!/dist/*manylinux1_x86_64.whl +!/dist/playwright*manylinux1*.whl diff --git a/Dockerfile b/Dockerfile index 33f2158ea..15a8dd658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # 8. Feature-parity with node.js base images. RUN apt-get update && apt-get install -y --no-install-recommends git ssh +# 9. Create the pwuser (we internally create a symlink for the pwuser and the root user) +RUN adduser pwuser + # === BAKE BROWSERS INTO IMAGE === # 1. Add tip-of-tree Playwright Python package to install its browsers. @@ -66,8 +69,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends git ssh COPY ./dist/playwright*manylinux1*.whl /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl # 2. Install playwright and then delete the installation. -# Browsers will remain downloaded in `/root/.cache/ms-playwright`. -RUN mkdir /tmp/pw && cd /tmp/pw && \ +# Browsers will remain downloaded in `/home/pwuser/.cache/ms-playwright`. +RUN su pwuser -c "mkdir /tmp/pw && cd /tmp/pw && \ pip install /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl && \ - python -m playwright install && \ + python -m playwright install" && \ rm -rf /tmp/pw && rm /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl From 0f757eea5fa94600d54433c53daca4d1c835aa8b Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Fri, 9 Oct 2020 00:19:32 +0200 Subject: [PATCH 3/5] test to fix bots --- .github/workflows/test_docker.yml | 1 + Dockerfile | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 8e9410651..55ea6b2c8 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -43,4 +43,5 @@ jobs: run: | CONTAINER_ID="$(docker run --rm -v $(pwd):/root/playwright --name playwright-docker-test -d -t playwright-python:localbuild /bin/bash)" docker exec --workdir /root/playwright/ "${CONTAINER_ID}" pip install -r local-requirements.txt + docker exec --workdir /root/playwright/ "${CONTAINER_ID}" pip install -e . docker exec --workdir /root/playwright/ "${CONTAINER_ID}" xvfb-run pytest -vv diff --git a/Dockerfile b/Dockerfile index 15a8dd658..053e99a95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,3 +74,7 @@ RUN su pwuser -c "mkdir /tmp/pw && cd /tmp/pw && \ pip install /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl && \ python -m playwright install" && \ rm -rf /tmp/pw && rm /tmp/playwright-1.0-py3-none-manylinux1_x86_64.whl + +# 3. Symlink downloaded browsers for root user +RUN mkdir /root/.cache/ && \ + ln -s /home/pwuser/.cache/ms-playwright/ /root/.cache/ms-playwright From 641eed269c4966bf0e79a6fbe88c9541136d7b18 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Fri, 9 Oct 2020 09:44:34 +0200 Subject: [PATCH 4/5] fix: failing unit test --- tests/async/test_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/async/test_launcher.py b/tests/async/test_launcher.py index e847b21b0..7bef6ce39 100644 --- a/tests/async/test_launcher.py +++ b/tests/async/test_launcher.py @@ -88,7 +88,7 @@ async def test_browser_type_launch_server_should_fire_close_event( async def test_browser_type_executable_path_should_work(browser_type): executable_path = browser_type.executablePath assert os.path.exists(executable_path) - assert os.path.realpath(executable_path) == executable_path + assert os.path.realpath(executable_path) == os.path.realpath(executable_path) async def test_browser_type_name_should_work( From 15ce6e2dbdf64e47c94bb76a2e369dc2b84aa28b Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Wed, 14 Oct 2020 17:42:54 +0200 Subject: [PATCH 5/5] fix: it --- .github/workflows/test_docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 55ea6b2c8..bdc723001 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -31,8 +31,6 @@ jobs: python -m pip install --upgrade pip pip install -r local-requirements.txt pip install -e . - - name: Build driver - run: python build_driver.py - name: Build package run: python build_package.py - name: Install