diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..c0a95e72 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [ + "optional-kubernetes-engine" + ] +} diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 00000000..7d8eca8e --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,40 @@ +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: true + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: false + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: true + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "kokoro" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: + # Team slug to add to repository permissions + - team: yoshi-admins + # Access level required, one of push|pull|admin + permission: admin + - team: python-samples-reviewers + permission: admin diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile index 2830f0af..accdd0bf 100644 --- a/.kokoro/docker/Dockerfile +++ b/.kokoro/docker/Dockerfile @@ -15,8 +15,8 @@ FROM gcr.io/cloud-devrel-kokoro-resources/python-base:latest # Install libraries needed by third-party python packages that we depend on. -RUN apt update \ - && apt install -y \ +RUN apt-get update \ + && apt-get install -y \ graphviz \ libcurl4-openssl-dev \ libffi-dev \ @@ -30,18 +30,19 @@ RUN apt update \ libxml2-dev \ libxslt1-dev \ openssl \ - portaudio19-dev \ - python-pyaudio \ zlib1g-dev \ - && apt clean + && apt-get clean -RUN python --version -RUN which python + +###################### Check python version + +RUN python3 --version +RUN which python3 # Setup Cloud SDK -ENV CLOUD_SDK_VERSION 324.0.0 +ENV CLOUD_SDK_VERSION 489.0.0 # Use system python for cloud sdk. -ENV CLOUDSDK_PYTHON /usr/bin/python +ENV CLOUDSDK_PYTHON python3.12 RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz RUN /google-cloud-sdk/install.sh @@ -51,6 +52,6 @@ ENV PATH /google-cloud-sdk/bin:$PATH ENV PATH ~/.local/bin:/root/.local/bin:$PATH # Install the current version of nox. -RUN python3 -m pip install --user --no-cache-dir nox==2020.12.31 +RUN python3 -m pip install --user --no-cache-dir nox==2022.1.7 CMD ["nox"] diff --git a/CODEOWNERS b/CODEOWNERS index 0480d02a..0dfefd93 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,4 +7,4 @@ # The python-samples-owners team is the default owner for anything not # explicitly taken by someone else. -* @GoogleCloudPlatform/python-samples-owners +* @GoogleCloudPlatform/python-samples-reviewers diff --git a/README.md b/README.md index 0e5b3292..a347e9bb 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ The code for the samples is contained in individual folders in this repository. Tutorial | Folder ---------|------- -[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/bookshelf) -[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/background) -[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/master/quickstart/python) -[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/gce) -[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/sessions) -[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/authenticating-users) +[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/bookshelf) +[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/background) +[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/quickstart/python) +[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/gce) +[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/sessions) +[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/authenticating-users) ## Contributing changes diff --git a/authenticating-users/main_test.py b/authenticating-users/main_test.py new file mode 100644 index 00000000..38b934b6 --- /dev/null +++ b/authenticating-users/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def fake_validate(assertion): + if assertion == "Valid": + return "nobody@example.com", "user0001" + else: + return None, None + + +main.validate_assertion = fake_validate + + +def test_home_page(): + client = main.app.test_client() + + # Good request check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Valid"}) + assert "nobody@example.com" in r.text + + # Missing header check + r = client.get("/") + assert "None" in r.text + + # Bad header check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Not Valid"}) + assert "None" in r.text diff --git a/authenticating-users/requirements-test.txt b/authenticating-users/requirements-test.txt new file mode 100644 index 00000000..6a3d7bca --- /dev/null +++ b/authenticating-users/requirements-test.txt @@ -0,0 +1 @@ +pytest==7.1.2 \ No newline at end of file diff --git a/authenticating-users/requirements.txt b/authenticating-users/requirements.txt index 693d0851..0d4a3c36 100644 --- a/authenticating-users/requirements.txt +++ b/authenticating-users/requirements.txt @@ -1,6 +1,6 @@ # [START getting_started_requirements] -Flask==1.1.2 -cryptography==3.3.1 -python-jose[cryptography]==3.2.0 -requests==2.25.1 +Flask==2.2.5 +cryptography==41.0.2 +python-jose[cryptography]==3.3.0 +requests==2.31.0 # [END getting_started_requirements] diff --git a/background/app/main_test.py b/background/app/main_test.py index f0dc9fe7..f3bf28c2 100644 --- a/background/app/main_test.py +++ b/background/app/main_test.py @@ -30,7 +30,7 @@ TOPIC_NAME = "projects/{}/topics/{}".format(project_id, "translate") -@pytest.yield_fixture +@pytest.fixture def db(): def clear_collection(collection): """Removes every document from the collection, to make it easy to see @@ -54,13 +54,13 @@ def clear_collection(collection): yield client -@pytest.yield_fixture +@pytest.fixture def publisher(): client = pubsub.PublisherClient() yield client -@pytest.yield_fixture +@pytest.fixture def subscriber(): subscriber = pubsub.SubscriberClient() subscriber.create_subscription( diff --git a/background/app/requirements.txt b/background/app/requirements.txt index 3ba144cd..7cfba806 100644 --- a/background/app/requirements.txt +++ b/background/app/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-firestore==2.0.2 -google-cloud-pubsub==2.2.0 -flask==1.1.2 +google-cloud-firestore==2.11.1 +google-cloud-pubsub==2.16.1 +flask==2.2.5 diff --git a/background/function/requirements.txt b/background/function/requirements.txt index f8b183e9..b8e6aaad 100644 --- a/background/function/requirements.txt +++ b/background/function/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-translate==3.0.2 -google-cloud-firestore==2.0.2 +google-cloud-translate==3.11.1 +google-cloud-firestore==2.11.1 diff --git a/bookshelf/Dockerfile b/bookshelf/Dockerfile new file mode 100644 index 00000000..44c98b38 --- /dev/null +++ b/bookshelf/Dockerfile @@ -0,0 +1,14 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.11-slim + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install --no-cache-dir -r requirements.txt + +# Run the web service on container startup. +ENTRYPOINT [ "gunicorn", "--bind", "0.0.0.0:8080", "main:app" ] \ No newline at end of file diff --git a/bookshelf/main_test.py b/bookshelf/main_test.py index 404de778..249cba17 100644 --- a/bookshelf/main_test.py +++ b/bookshelf/main_test.py @@ -26,7 +26,7 @@ os.environ['GOOGLE_CLOUD_PROJECT'] = project_id -@pytest.yield_fixture +@pytest.fixture def app(request): """This fixture provides a Flask app instance configured for testing. @@ -38,17 +38,17 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def firestore(): + """This fixture provides a modified version of the app's Firebase model + that tracks all created items and deletes them at the end of the test. - import firestore - """This fixture provides a modified version of the app's Firebase model that - tracks all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. + Any tests that directly or indirectly interact with the database should + use this to ensure that resources are properly cleaned up. """ + import firestore + # Ensure no books exist before running the tests. This typically helps if # tests somehow left the database in a bad state. delete_all_books(firestore) diff --git a/bookshelf/requirements.txt b/bookshelf/requirements.txt index 927e9c97..27d3d4fe 100644 --- a/bookshelf/requirements.txt +++ b/bookshelf/requirements.txt @@ -1,7 +1,7 @@ -Flask==1.1.2 -google-cloud-firestore==2.0.2 -google-cloud-storage==1.35.0 -google-cloud-logging==2.1.1 -google-cloud-error-reporting==1.1.0 -gunicorn==20.0.4 -six==1.15.0 +Flask==2.2.5 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.9.0 +google-cloud-error-reporting==1.9.1 +google-cloud-logging==3.5.0 +gunicorn==20.1.0 +six==1.16.0 diff --git a/gce/add-google-cloud-ops-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh new file mode 100644 index 00000000..63cf2dbd --- /dev/null +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -0,0 +1,519 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *NOTE*: The source of truth for this script is: +# https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +# See https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/installation +# for installation instructions. +# It is committed to this repository to follow security best practices. +# +# +# Add repository for the Google ops agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--uninstall-standalone-logging-agent`: +# Uninstalls the standalone logging agent (`google-fluentd`). +# +# * `--uninstall-standalone-monitoring-agent`: +# Uninstalls the standalone monitoring agent (`stackdriver-agent`). +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-google-cloud-ops-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --dry-run +# +# * To replace standalone agents with the Ops agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --uninstall-standalone-logging-agent --uninstall-standalone-monitoring-agent +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-ops-agent-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + uninstall-standalone-logging-agent) ACTIONS+=('uninstall-standalone-logging-agent') ;; + uninstall-standalone-monitoring-agent) ACTIONS+=('uninstall-standalone-monitoring-agent') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ " ${ACTIONS[*]} " == *\ uninstall\ * || ( " ${ACTIONS[*]} " == *\ remove-repo\ * && " ${ACTIONS[*]} " != *\ also-install\ * )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE + +if [[ " ${ACTIONS[*]} " == *\ also-install*uninstall\ * ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the ops agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/stackdriver/docs/solutions/ops-agent' + +# URL documentation which lists supported platforms for running the ops agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#supported_operating_systems" + +# Packages to install. +AGENT_PACKAGE='google-cloud-ops-agent' +declare -a ADDITIONAL_PACKAGES=() + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-ops-agent ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-ops-agent.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-ops-agent.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-ops-agent.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-ops-agent.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 2.0.1~debian9.13. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}~*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early unless at least one package is installed. + dpkg -l "${packages[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-ops-agent.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-ops-agent/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-ops-agent || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +save_configuration_files() { + local save_dir="/var/lib/google-cloud-ops-agent/saved_configs" + ${DRY_RUN} mkdir -p "${save_dir}" + ${DRY_RUN} cp -rp "$@" "${save_dir}" + echo "$* is now copied over to ${save_dir} folder." +} + +main() { + case "${ID:-}" in + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-logging-agent\ * ]]; then + save_configuration_files "/etc/google-fluentd" + # This will also remove dependent packages, e.g. "google-fluentd-catch-all-config" or "google-fluentd-catch-all-config-structured". + uninstall "google-fluentd" + fi + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-monitoring-agent\ * ]]; then + save_configuration_files "/etc/stackdriver" "/opt/stackdriver/collectd/etc" + uninstall "stackdriver-agent" + fi + if [[ " ${ACTIONS[*]} " == *\ add-repo\ * ]]; then + resolve_version + add_repo + fi + if [[ " ${ACTIONS[*]} " == *\ also-install\ * ]]; then + install_agent + elif [[ " ${ACTIONS[*]} " == *\ uninstall\ * ]]; then + save_configuration_files "/etc/google-cloud-ops-agent" + uninstall "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" + fi + if [[ " ${ACTIONS[*]} " == *\ remove-repo\ * ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/deploy.sh b/gce/deploy.sh index b3449837..36ed75ca 100644 --- a/gce/deploy.sh +++ b/gce/deploy.sh @@ -19,7 +19,7 @@ MY_INSTANCE_NAME="my-app-instance" ZONE=us-central1-a gcloud compute instances create $MY_INSTANCE_NAME \ - --image-family=debian-9 \ + --image-family=debian-10 \ --image-project=debian-cloud \ --machine-type=g1-small \ --scopes userinfo-email,cloud-platform \ diff --git a/gce/main_test.py b/gce/main_test.py new file mode 100644 index 00000000..6d1f887b --- /dev/null +++ b/gce/main_test.py @@ -0,0 +1,25 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_hello(): + main.app.testing = True + client = main.app.test_client() + + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Hello, world!" in response_text diff --git a/gce/requirements.txt b/gce/requirements.txt index 1eb18140..b655465f 100644 --- a/gce/requirements.txt +++ b/gce/requirements.txt @@ -1,3 +1,3 @@ -flask==1.1.2 -honcho==1.0.1 -gunicorn==20.0.4 +flask==2.2.5 +honcho==1.1.0 +gunicorn==20.1.0 diff --git a/gce/startup-script.sh b/gce/startup-script.sh index 2068606e..cc35d30f 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -12,29 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Echo commands -set -v +# Echo commands and fail on error +set -ev # [START getting_started_gce_startup_script] -# Install Stackdriver logging agent -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -sudo bash install-logging-agent.sh - # Install or update needed software apt-get update -apt-get install -yq git supervisor python python-pip +apt-get install -yq git supervisor python python-pip python3-distutils pip install --upgrade pip virtualenv -# Account to own server process -useradd -m -d /home/pythonapp pythonapp - # Fetch source code export HOME=/root git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app +# Install Cloud Ops Agent +sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install + +# Account to own server process +useradd -m -d /home/pythonapp pythonapp + # Python environment setup virtualenv -p python3 /opt/app/gce/env -source /opt/app/gce/env/bin/activate +/bin/bash -c "source /opt/app/gce/env/bin/activate" /opt/app/gce/env/bin/pip install -r /opt/app/gce/requirements.txt # Set ownership to newly created account diff --git a/gce/teardown.sh b/gce/teardown.sh index b31d1139..668b1218 100644 --- a/gce/teardown.sh +++ b/gce/teardown.sh @@ -16,8 +16,8 @@ set -x -MY_INSTANCE_NAME="my-instance-name" -ZONE=us-central1-f +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a gcloud compute instances delete $MY_INSTANCE_NAME \ --zone=$ZONE --delete-disks=all diff --git a/noxfile.py b/noxfile.py index 8d7c7d5e..8f5a921c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -55,7 +55,7 @@ def run_test(session, dir): success_codes=[0, 5]) -@nox.session +@nox.session(python="3.12") @nox.parametrize('dir', DIRS) def run_tests(session, dir=None): """Run all tests for all directories (slow!)""" diff --git a/optional-kubernetes-engine/.dockerignore b/optional-kubernetes-engine/.dockerignore index fdc976d4..c4730648 100644 --- a/optional-kubernetes-engine/.dockerignore +++ b/optional-kubernetes-engine/.dockerignore @@ -12,6 +12,9 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log .git +.mypy_cache +.pytest_cache +.hypothesis diff --git a/optional-kubernetes-engine/tests/conftest.py b/optional-kubernetes-engine/tests/conftest.py index d098db9d..57a50a1c 100644 --- a/optional-kubernetes-engine/tests/conftest.py +++ b/optional-kubernetes-engine/tests/conftest.py @@ -22,7 +22,7 @@ from retrying import retry -@pytest.yield_fixture(params=['datastore', 'mongodb']) +@pytest.fixture(params=['datastore', 'mongodb']) def app(request): """This fixtures provides a Flask app instance configured for testing. @@ -43,7 +43,7 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def model(monkeypatch, app): """This fixture provides a modified version of the app's model that tracks all created items and deletes them at the end of the test. diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 07729f34..00000000 --- a/renovate.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": [ - "config:base" - ], - "ignorePaths": [ - "optional-kubernetes-engine" - ] -} diff --git a/requirements.txt b/requirements.txt index 43cd9314..0fd66f54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -flake8==3.8.4 -nox==2020.12.31 -pytest==6.2.1 -requests==2.25.1 +flake8===5.0.4; python_version < '3.8' +flake8==6.0.0; python_version >= '3.8' +pytest==7.3.1 +nox==2023.4.22 +requests==2.31.0 diff --git a/sessions/requirements.txt b/sessions/requirements.txt index 43c7ea10..e0e267b5 100644 --- a/sessions/requirements.txt +++ b/sessions/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-firestore==2.0.2 -flask==1.1.2 +google-cloud-firestore==2.11.1 +flask==2.2.5