diff --git a/.gitignore b/.gitignore index b87e1ed..b9daa52 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ pip-log.txt # Built documentation docs/_build bigquery/docs/generated +docs.metadata # Virtual environment env/ @@ -57,4 +58,4 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc -pylintrc.test \ No newline at end of file +pylintrc.test diff --git a/.kokoro/build.sh b/.kokoro/build.sh index cd6ab9c..35d44fd 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,4 +36,10 @@ python3.6 -m pip uninstall --yes --quiet nox-automation python3.6 -m pip install --upgrade --quiet nox python3.6 -m nox --version -python3.6 -m nox +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3.6 -m nox -s "${NOX_SESSION:-}" +else + python3.6 -m nox +fi diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile new file mode 100644 index 0000000..412b0b5 --- /dev/null +++ b/.kokoro/docker/docs/Dockerfile @@ -0,0 +1,98 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ubuntu:20.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Ensure local Python is preferred over distribution Python. +ENV PATH /usr/local/bin:$PATH + +# Install dependencies. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + git \ + gpg-agent \ + graphviz \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libsnappy-dev \ + libssl-dev \ + libsqlite3-dev \ + portaudio19-dev \ + redis-server \ + software-properties-common \ + ssh \ + sudo \ + tcl \ + tcl-dev \ + tk \ + tk-dev \ + uuid-dev \ + wget \ + zlib1g-dev \ + && add-apt-repository universe \ + && apt-get update \ + && apt-get -y install jq \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + + +COPY fetch_gpg_keys.sh /tmp +# Install the desired versions of Python. +RUN set -ex \ + && export GNUPGHOME="$(mktemp -d)" \ + && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ + && /tmp/fetch_gpg_keys.sh \ + && for PYTHON_VERSION in 3.7.8 3.8.5; do \ + wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ + && rm -r python-${PYTHON_VERSION}.tar.xz.asc \ + && mkdir -p /usr/src/python-${PYTHON_VERSION} \ + && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ + && rm python-${PYTHON_VERSION}.tar.xz \ + && cd /usr/src/python-${PYTHON_VERSION} \ + && ./configure \ + --enable-shared \ + # This works only on Python 2.7 and throws a warning on every other + # version, but seems otherwise harmless. + --enable-unicode=ucs4 \ + --with-system-ffi \ + --without-ensurepip \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + ; done \ + && rm -rf "${GNUPGHOME}" \ + && rm -rf /usr/src/python* \ + && rm -rf ~/.cache/ + +RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3.7 /tmp/get-pip.py \ + && python3.8 /tmp/get-pip.py \ + && rm /tmp/get-pip.py + +CMD ["python3.7"] diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh new file mode 100755 index 0000000..d653dd8 --- /dev/null +++ b/.kokoro/docker/docs/fetch_gpg_keys.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A script to fetch gpg keys with retry. +# Avoid jinja parsing the file. +# + +function retry { + if [[ "${#}" -le 1 ]]; then + echo "Usage: ${0} retry_count commands.." + exit 1 + fi + local retries=${1} + local command="${@:2}" + until [[ "${retries}" -le 0 ]]; do + $command && return 0 + if [[ $? -ne 0 ]]; then + echo "command failed, retrying" + ((retries--)) + fi + done + return 1 +} + +# 3.6.9, 3.7.5 (Ned Deily) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D + +# 3.8.0 (Łukasz Langa) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + E3FF2839C048B25C084DEBE9B26995E310250568 + +# diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 88fa5e1..d68fb9e 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -11,12 +11,12 @@ action { gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-containeranalysis/.kokoro/trampoline.sh" +build_file: "python-containeranalysis/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" + value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" } env_vars: { key: "TRAMPOLINE_BUILD_FILE" @@ -28,6 +28,23 @@ env_vars: { value: "docs-staging" } +env_vars: { + key: "V2_STAGING_BUCKET" + value: "docs-staging-v2-staging" +} + +# It will upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} + +# It will always build the docker image. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/docs/Dockerfile" +} + # Fetch the token needed for reporting release status to GitHub before_action { fetch_keystore { diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg new file mode 100644 index 0000000..1118107 --- /dev/null +++ b/.kokoro/docs/docs-presubmit.cfg @@ -0,0 +1,17 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "STAGING_BUCKET" + value: "gcloud-python-test" +} + +env_vars: { + key: "V2_STAGING_BUCKET" + value: "gcloud-python-test" +} + +# We only upload the image in the main `docs` build. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "false" +} diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 8b901b0..8acb14e 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -18,26 +18,16 @@ set -eo pipefail # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 -cd github/python-containeranalysis - -# Remove old nox -python3.6 -m pip uninstall --yes --quiet nox-automation +export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3.6 -m pip install --upgrade --quiet nox -python3.6 -m nox --version +python3 -m pip install --user --upgrade --quiet nox +python3 -m nox --version # build docs nox -s docs -python3 -m pip install gcp-docuploader - -# install a json parser -sudo apt-get update -sudo apt-get -y install software-properties-common -sudo add-apt-repository universe -sudo apt-get update -sudo apt-get -y install jq +python3 -m pip install --user gcp-docuploader # create metadata python3 -m docuploader create-metadata \ @@ -52,4 +42,23 @@ python3 -m docuploader create-metadata \ cat docs.metadata # upload docs -python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" + + +# docfx yaml files +nox -s docfx + +# create metadata. +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 0000000..719bcd5 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Build Cop Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 0000000..995ee29 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f9834..d84aec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [1.0.3](https://www.github.com/googleapis/python-containeranalysis/compare/v1.0.2...v1.0.3) (2020-08-11) + + +### Bug Fixes + +* Use different versions of pytest for python 2 and python3 [([#2558](https://www.github.com/googleapis/python-containeranalysis/issues/2558))](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2558) ([7d21641](https://www.github.com/googleapis/python-containeranalysis/commit/7d21641eb50f574784ae7dfbb1d25a0d0af14699)) +* **deps:** add upper bound for grafeas ([#30](https://www.github.com/googleapis/python-containeranalysis/issues/30)) ([5ca8f79](https://www.github.com/googleapis/python-containeranalysis/commit/5ca8f7981349ed86438185f02681225f059cc9d9)) + ### [1.0.2](https://www.github.com/googleapis/python-containeranalysis/compare/v1.0.1...v1.0.2) (2020-07-16) diff --git a/docs/conf.py b/docs/conf.py index 58c9eb8..9fb6f92 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("..")) +# For plugins that can not read conf.py. +# See also: https://github.com/docascode/sphinx-docfx-yaml/issues/85 +sys.path.insert(0, os.path.abspath(".")) + __version__ = "" # -- General configuration ------------------------------------------------ @@ -90,7 +94,12 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = [ + "_build", + "samples/AUTHORING_GUIDE.md", + "samples/CONTRIBUTING.md", + "samples/snippets/README.rst", +] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/noxfile.py b/noxfile.py index 39ee5d8..fcfde61 100644 --- a/noxfile.py +++ b/noxfile.py @@ -100,6 +100,10 @@ def system(session): """Run the system test suite.""" system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") # Sanity check: Only run tests if the environment variable is set. if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): session.skip("Credentials must be set via environment variable") @@ -160,3 +164,36 @@ def docs(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install("sphinx<3.0.0", "alabaster", "recommonmark", "sphinx-docfx-yaml") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md new file mode 100644 index 0000000..55c97b3 --- /dev/null +++ b/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md new file mode 100644 index 0000000..34c882b --- /dev/null +++ b/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/snippets/.gitignore b/samples/snippets/.gitignore new file mode 100644 index 0000000..9e3d04c --- /dev/null +++ b/samples/snippets/.gitignore @@ -0,0 +1 @@ +venv* diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 0000000..73c45c3 --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,54 @@ +Google +Cloud Platform logo + +# Google Cloud Container Analysis Samples + + +Container Analysis scans container images stored in Container Registry for vulnerabilities. +Continuous automated analysis of containers keep you informed about known vulnerabilities so +that you can review and address issues before deployment. + +Additionally, third-party metadata providers can use Container Analysis to store and +retrieve additional metadata for their customers' images, such as packages installed in an image. + + +## Description + +These samples show how to use the [Google Cloud Container Analysis Client Library](https://cloud.google.com/container-registry/docs/reference/libraries). + +## Build and Run +1. **Enable APIs** + - [Enable the Container Analysis API](https://console.cloud.google.com/flows/enableapi?apiid=containeranalysis.googleapis.com) + and create a new project or select an existing project. +1. **Install and Initialize Cloud SDK** + - Follow instructions from the available [quickstarts](https://cloud.google.com/sdk/docs/quickstarts) +1. **Authenticate with GCP** + - Typically, you should authenticate using a [service account key](https://cloud.google.com/docs/authentication/getting-started) +1. **Clone the repo** and cd into this directory + + ``` + git clone https://github.com/GoogleCloudPlatform/python-docs-samples + cd python-docs-samples + ``` + +1. **Set Environment Variables** + + ``` + export GCLOUD_PROJECT="YOUR_PROJECT_ID" + ``` + +1. **Run Tests** + + ``` + nox -s "py36(sample='./container_registry/container_analysis')" + ``` + +## Contributing changes + +* See [CONTRIBUTING.md](../../CONTRIBUTING.md) + +## Licensing + +* See [LICENSE](../../LICENSE) + diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py new file mode 100644 index 0000000..ba55d7c --- /dev/null +++ b/samples/snippets/noxfile.py @@ -0,0 +1,224 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append('.') + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG['gcloud_project_env'] + # This should error out if not set. + ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG['envs']) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + "." + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip("SKIPPED: {} tests are disabled for this sample.".format( + session.python + )) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt new file mode 100644 index 0000000..7e460c8 --- /dev/null +++ b/samples/snippets/requirements-test.txt @@ -0,0 +1 @@ +pytest==6.0.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 0000000..865f9ba --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1,7 @@ +google-cloud-pubsub==1.7.0 +google-cloud-containeranalysis==1.0.2 +grafeas==0.4.1 +pytest==5.3.0; python_version > "3.0" +pytest==4.6.6; python_version < "3.0" +flaky==3.7.0 +mock==4.0.2 diff --git a/samples/snippets/samples.py b/samples/snippets/samples.py new file mode 100644 index 0000000..7d1348c --- /dev/null +++ b/samples/snippets/samples.py @@ -0,0 +1,373 @@ +#!/bin/python +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START containeranalysis_create_note] +def create_note(note_id, project_id): + """Creates and returns a new vulnerability note.""" + # note_id = 'my-note' + # project_id = 'my-gcp-project' + + from grafeas.grafeas_v1.gapic.enums import Version + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + note = { + 'vulnerability': { + 'details': [ + { + 'affected_cpe_uri': 'your-uri-here', + 'affected_package': 'your-package-here', + 'affected_version_start': { + 'kind': Version.VersionKind.MINIMUM + }, + 'fixed_version': { + 'kind': Version.VersionKind.MAXIMUM + } + } + ] + } + } + response = grafeas_client.create_note(project_name, note_id, note) + return response +# [END containeranalysis_create_note] + + +# [START containeranalysis_delete_note] +def delete_note(note_id, project_id): + """Removes an existing note from the server.""" + # note_id = 'my-note' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + note_name = f"projects/{project_id}/notes/{note_id}" + + grafeas_client.delete_note(note_name) +# [END containeranalysis_delete_note] + + +# [START containeranalysis_create_occurrence] +def create_occurrence(resource_url, note_id, occurrence_project, note_project): + """ Creates and returns a new occurrence of a previously + created vulnerability note.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # note_id = 'my-note' + # occurrence_project = 'my-gcp-project' + # note_project = 'my-gcp-project' + + from grafeas.grafeas_v1.gapic.enums import Version + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + formatted_note = f"projects/{note_project}/notes/{note_id}" + formatted_project = grafeas_client.project_path(occurrence_project) + + occurrence = { + 'note_name': formatted_note, + 'resource_uri': resource_url, + 'vulnerability': { + 'package_issue': [ + { + 'affected_cpe_uri': 'your-uri-here', + 'affected_package': 'your-package-here', + 'affected_version': { + 'kind': Version.VersionKind.MINIMUM + }, + 'fixed_version': { + 'kind': Version.VersionKind.MAXIMUM + } + } + ] + } + } + + return grafeas_client.create_occurrence(formatted_project, occurrence) +# [END containeranalysis_create_occurrence] + + +# [START containeranalysis_delete_occurrence] +def delete_occurrence(occurrence_id, project_id): + """Removes an existing occurrence from the server.""" + # occurrence_id = basename(occurrence.name) + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + parent = f"projects/{project_id}/occurrences/{occurrence_id}" + grafeas_client.delete_occurrence(parent) +# [END containeranalysis_delete_occurrence] + + +# [START containeranalysis_get_note] +def get_note(note_id, project_id): + """Retrieves and prints a specified note from the server.""" + # note_id = 'my-note' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + note_name = f"projects/{project_id}/notes/{note_id}" + response = grafeas_client.get_note(note_name) + return response +# [END containeranalysis_get_note] + + +# [START containeranalysis_get_occurrence] +def get_occurrence(occurrence_id, project_id): + """retrieves and prints a specified occurrence from the server.""" + # occurrence_id = basename(occurrence.name) + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + parent = f"projects/{project_id}/occurrences/{occurrence_id}" + return grafeas_client.get_occurrence(parent) +# [END containeranalysis_get_occurrence] + + +# [START containeranalysis_discovery_info] +def get_discovery_info(resource_url, project_id): + """Retrieves and prints the discovery occurrence created for a specified + image. The discovery occurrence contains information about the initial + scan on the image.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + filter_str = 'kind="DISCOVERY" AND resourceUrl="{}"'.format(resource_url) + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + response = grafeas_client.list_occurrences(project_name, + filter_=filter_str) + for occ in response: + print(occ) +# [END containeranalysis_discovery_info] + + +# [START containeranalysis_occurrences_for_note] +def get_occurrences_for_note(note_id, project_id): + """Retrieves all the occurrences associated with a specified Note. + Here, all occurrences are printed and counted.""" + # note_id = 'my-note' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + note_name = f"projects/{project_id}/notes/{note_id}" + + response = grafeas_client.list_note_occurrences(note_name) + count = 0 + for o in response: + # do something with the retrieved occurrence + # in this sample, we will simply count each one + count += 1 + return count +# [END containeranalysis_occurrences_for_note] + + +# [START containeranalysis_occurrences_for_image] +def get_occurrences_for_image(resource_url, project_id): + """Retrieves all the occurrences associated with a specified image. + Here, all occurrences are simply printed and counted.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + filter_str = 'resourceUrl="{}"'.format(resource_url) + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + + response = grafeas_client.list_occurrences(project_name, + filter_=filter_str) + count = 0 + for o in response: + # do something with the retrieved occurrence + # in this sample, we will simply count each one + count += 1 + return count +# [END containeranalysis_occurrences_for_image] + + +# [START containeranalysis_pubsub] +def pubsub(subscription_id, timeout_seconds, project_id): + """Respond to incoming occurrences using a Cloud Pub/Sub subscription.""" + # subscription_id := 'my-occurrences-subscription' + # timeout_seconds = 20 + # project_id = 'my-gcp-project' + + import time + from google.cloud.pubsub import SubscriberClient + + client = SubscriberClient() + subscription_name = client.subscription_path(project_id, subscription_id) + receiver = MessageReceiver() + client.subscribe(subscription_name, receiver.pubsub_callback) + + # listen for 'timeout' seconds + for _ in range(timeout_seconds): + time.sleep(1) + # print and return the number of pubsub messages received + print(receiver.msg_count) + return receiver.msg_count + + +class MessageReceiver: + """Custom class to handle incoming Pub/Sub messages.""" + def __init__(self): + # initialize counter to 0 on initialization + self.msg_count = 0 + + def pubsub_callback(self, message): + # every time a pubsub message comes in, print it and count it + self.msg_count += 1 + print('Message {}: {}'.format(self.msg_count, message.data)) + message.ack() + + +def create_occurrence_subscription(subscription_id, project_id): + """Creates a new Pub/Sub subscription object listening to the + Container Analysis Occurrences topic.""" + # subscription_id := 'my-occurrences-subscription' + # project_id = 'my-gcp-project' + + from google.api_core.exceptions import AlreadyExists + from google.cloud.pubsub import SubscriberClient + + topic_id = 'container-analysis-occurrences-v1' + client = SubscriberClient() + topic_name = client.topic_path(project_id, topic_id) + subscription_name = client.subscription_path(project_id, subscription_id) + success = True + try: + client.create_subscription(subscription_name, topic_name) + except AlreadyExists: + # if subscription already exists, do nothing + pass + else: + success = False + return success +# [END containeranalysis_pubsub] + + +# [START containeranalysis_poll_discovery_occurrence_finished] +def poll_discovery_finished(resource_url, timeout_seconds, project_id): + """Returns the discovery occurrence for a resource once it reaches a + terminal state.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # timeout_seconds = 20 + # project_id = 'my-gcp-project' + + import time + from grafeas.grafeas_v1.gapic.enums import DiscoveryOccurrence + from google.cloud.devtools import containeranalysis_v1 + + deadline = time.time() + timeout_seconds + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + + discovery_occurrence = None + while discovery_occurrence is None: + time.sleep(1) + filter_str = 'resourceUrl="{}" \ + AND noteProjectId="goog-analysis" \ + AND noteId="PACKAGE_VULNERABILITY"'.format(resource_url) + # [END containeranalysis_poll_discovery_occurrence_finished] + # The above filter isn't testable, since it looks for occurrences in a + # locked down project fall back to a more permissive filter for testing + filter_str = 'kind="DISCOVERY" AND resourceUrl="{}"'\ + .format(resource_url) + # [START containeranalysis_poll_discovery_occurrence_finished] + result = grafeas_client.list_occurrences(project_name, filter_str) + # only one occurrence should ever be returned by ListOccurrences + # and the given filter + for item in result: + discovery_occurrence = item + if time.time() > deadline: + raise RuntimeError('timeout while retrieving discovery occurrence') + + status = DiscoveryOccurrence.AnalysisStatus.PENDING + while status != DiscoveryOccurrence.AnalysisStatus.FINISHED_UNSUPPORTED \ + and status != DiscoveryOccurrence.AnalysisStatus.FINISHED_FAILED \ + and status != DiscoveryOccurrence.AnalysisStatus.FINISHED_SUCCESS: + time.sleep(1) + updated = grafeas_client.get_occurrence(discovery_occurrence.name) + status = updated.discovery.analysis_status + if time.time() > deadline: + raise RuntimeError('timeout while waiting for terminal state') + return discovery_occurrence +# [END containeranalysis_poll_discovery_occurrence_finished] + + +# [START containeranalysis_vulnerability_occurrences_for_image] +def find_vulnerabilities_for_image(resource_url, project_id): + """"Retrieves all vulnerability occurrences associated with a resource.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # project_id = 'my-gcp-project' + + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + + filter_str = 'kind="VULNERABILITY" AND resourceUrl="{}"'\ + .format(resource_url) + return list(grafeas_client.list_occurrences(project_name, filter_str)) +# [END containeranalysis_vulnerability_occurrences_for_image] + + +# [START containeranalysis_filter_vulnerability_occurrences] +def find_high_severity_vulnerabilities_for_image(resource_url, project_id): + """Retrieves a list of only high vulnerability occurrences associated + with a resource.""" + # resource_url = 'https://gcr.io/my-project/my-image@sha256:123' + # project_id = 'my-gcp-project' + + from grafeas.grafeas_v1.gapic.enums import Severity + from google.cloud.devtools import containeranalysis_v1 + + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + project_name = grafeas_client.project_path(project_id) + + filter_str = 'kind="VULNERABILITY" AND resourceUrl="{}"'\ + .format(resource_url) + vulnerabilities = grafeas_client.list_occurrences(project_name, filter_str) + filtered_list = [] + for v in vulnerabilities: + if v.effective_severity == Severity.HIGH or v.effective_severity == Severity.CRITICAL: + filtered_list.append(v) + return filtered_list +# [END containeranalysis_filter_vulnerability_occurrences] diff --git a/samples/snippets/samples_test.py b/samples/snippets/samples_test.py new file mode 100644 index 0000000..56ac9ab --- /dev/null +++ b/samples/snippets/samples_test.py @@ -0,0 +1,316 @@ +#!/bin/python +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import environ +from os.path import basename +import threading +import time +import uuid + +from google.api_core.exceptions import AlreadyExists +from google.api_core.exceptions import InvalidArgument +from google.api_core.exceptions import NotFound +from google.cloud.devtools import containeranalysis_v1 +from google.cloud.pubsub import PublisherClient, SubscriberClient + +from grafeas.grafeas_v1.gapic.enums import DiscoveryOccurrence +from grafeas.grafeas_v1.gapic.enums import NoteKind +from grafeas.grafeas_v1.gapic.enums import Severity +from grafeas.grafeas_v1.gapic.enums import Version +import pytest + +import samples + +PROJECT_ID = environ['GOOGLE_CLOUD_PROJECT'] +SLEEP_TIME = 1 +TRY_LIMIT = 20 + + +class MessageReceiver: + """Custom class to handle incoming Pub/Sub messages.""" + def __init__(self, expected_msg_nums, done_event): + # initialize counter to 0 on initialization + self.msg_count = 0 + self.expected_msg_nums = expected_msg_nums + self.done_event = done_event + + def pubsub_callback(self, message): + # every time a pubsub message comes in, print it and count it + self.msg_count += 1 + print('Message {}: {}'.format(self.msg_count, message.data)) + message.ack() + if (self.msg_count == self.expected_msg_nums): + self.done_event.set() + + +class TestContainerAnalysisSamples: + + def setup_method(self, test_method): + print('SETUP {}'.format(test_method.__name__)) + self.note_id = 'note-{}'.format(uuid.uuid4()) + self.image_url = '{}.{}'.format(uuid.uuid4(), test_method.__name__) + self.note_obj = samples.create_note(self.note_id, PROJECT_ID) + + def teardown_method(self, test_method): + print('TEAR DOWN {}'.format(test_method.__name__)) + try: + samples.delete_note(self.note_id, PROJECT_ID) + except NotFound: + pass + + def test_create_note(self): + new_note = samples.get_note(self.note_id, PROJECT_ID) + assert new_note.name == self.note_obj.name + + def test_delete_note(self): + samples.delete_note(self.note_id, PROJECT_ID) + try: + samples.get_note(self.note_obj, PROJECT_ID) + except InvalidArgument: + pass + else: + # didn't raise exception we expected + assert (False) + + def test_create_occurrence(self): + created = samples.create_occurrence(self.image_url, + self.note_id, + PROJECT_ID, + PROJECT_ID) + retrieved = samples.get_occurrence(basename(created.name), PROJECT_ID) + assert created.name == retrieved.name + # clean up + samples.delete_occurrence(basename(created.name), PROJECT_ID) + + def test_delete_occurrence(self): + created = samples.create_occurrence(self.image_url, + self.note_id, + PROJECT_ID, + PROJECT_ID) + samples.delete_occurrence(basename(created.name), PROJECT_ID) + try: + samples.get_occurrence(basename(created.name), PROJECT_ID) + except NotFound: + pass + else: + # didn't raise exception we expected + assert False + + def test_occurrences_for_image(self): + orig_count = samples.get_occurrences_for_image(self.image_url, + PROJECT_ID) + occ = samples.create_occurrence(self.image_url, + self.note_id, + PROJECT_ID, + PROJECT_ID) + new_count = 0 + tries = 0 + while new_count != 1 and tries < TRY_LIMIT: + tries += 1 + new_count = samples.get_occurrences_for_image(self.image_url, + PROJECT_ID) + time.sleep(SLEEP_TIME) + assert new_count == 1 + assert orig_count == 0 + # clean up + samples.delete_occurrence(basename(occ.name), PROJECT_ID) + + def test_occurrences_for_note(self): + orig_count = samples.get_occurrences_for_note(self.note_id, + PROJECT_ID) + occ = samples.create_occurrence(self.image_url, + self.note_id, + PROJECT_ID, + PROJECT_ID) + new_count = 0 + tries = 0 + while new_count != 1 and tries < TRY_LIMIT: + tries += 1 + new_count = samples.get_occurrences_for_note(self.note_id, + PROJECT_ID) + time.sleep(SLEEP_TIME) + assert new_count == 1 + assert orig_count == 0 + # clean up + samples.delete_occurrence(basename(occ.name), PROJECT_ID) + + @pytest.mark.flaky(max_runs=3, min_passes=1) + def test_pubsub(self): + # create topic if needed + client = SubscriberClient() + try: + topic_id = 'container-analysis-occurrences-v1' + topic_name = client.topic_path(PROJECT_ID, topic_id) + publisher = PublisherClient() + publisher.create_topic(topic_name) + except AlreadyExists: + pass + + subscription_id = 'container-analysis-test-{}'.format(uuid.uuid4()) + subscription_name = client.subscription_path(PROJECT_ID, + subscription_id) + samples.create_occurrence_subscription(subscription_id, PROJECT_ID) + + # I can not make it pass with multiple messages. My guess is + # the server started to dedup? + message_count = 1 + try: + job_done = threading.Event() + receiver = MessageReceiver(message_count, job_done) + client.subscribe(subscription_name, receiver.pubsub_callback) + + for i in range(message_count): + occ = samples.create_occurrence( + self.image_url, self.note_id, PROJECT_ID, PROJECT_ID) + time.sleep(SLEEP_TIME) + samples.delete_occurrence(basename(occ.name), PROJECT_ID) + time.sleep(SLEEP_TIME) + # We saw occational failure with 60 seconds timeout, so we bumped it + # to 180 seconds. + # See also: python-docs-samples/issues/2894 + job_done.wait(timeout=180) + print('done. msg_count = {}'.format(receiver.msg_count)) + assert message_count <= receiver.msg_count + finally: + # clean up + client.delete_subscription(subscription_name) + + def test_poll_discovery_occurrence(self): + # try with no discovery occurrence + try: + samples.poll_discovery_finished(self.image_url, 5, PROJECT_ID) + except RuntimeError: + pass + else: + # we expect timeout error + assert False + + # create discovery occurrence + note_id = 'discovery-note-{}'.format(uuid.uuid4()) + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + note = { + 'discovery': { + 'analysis_kind': NoteKind.DISCOVERY + } + } + grafeas_client.\ + create_note(grafeas_client.project_path(PROJECT_ID), note_id, note) + occurrence = { + 'note_name': f"projects/{PROJECT_ID}/notes/{note_id}", + 'resource_uri': self.image_url, + 'discovery': { + 'analysis_status': DiscoveryOccurrence.AnalysisStatus + .FINISHED_SUCCESS + } + } + created = grafeas_client.\ + create_occurrence(grafeas_client.project_path(PROJECT_ID), + occurrence) + + # poll again + disc = samples.poll_discovery_finished(self.image_url, 10, PROJECT_ID) + status = disc.discovery.analysis_status + assert disc is not None + assert status == DiscoveryOccurrence.AnalysisStatus.FINISHED_SUCCESS + + # clean up + samples.delete_occurrence(basename(created.name), PROJECT_ID) + samples.delete_note(note_id, PROJECT_ID) + + def test_find_vulnerabilities_for_image(self): + occ_list = samples.find_vulnerabilities_for_image(self.image_url, + PROJECT_ID) + assert len(occ_list) == 0 + + created = samples.create_occurrence(self.image_url, + self.note_id, + PROJECT_ID, + PROJECT_ID) + tries = 0 + count = 0 + while count != 1 and tries < TRY_LIMIT: + tries += 1 + occ_list = samples.find_vulnerabilities_for_image(self.image_url, + PROJECT_ID) + count = len(occ_list) + time.sleep(SLEEP_TIME) + assert len(occ_list) == 1 + samples.delete_occurrence(basename(created.name), PROJECT_ID) + + def test_find_high_severity_vulnerabilities(self): + occ_list = samples.find_high_severity_vulnerabilities_for_image( + self.image_url, + PROJECT_ID) + assert len(occ_list) == 0 + + # create new high severity vulnerability + note_id = 'discovery-note-{}'.format(uuid.uuid4()) + client = containeranalysis_v1.ContainerAnalysisClient() + grafeas_client = client.get_grafeas_client() + note = { + 'vulnerability': { + 'severity': Severity.CRITICAL, + 'details': [ + { + 'affected_cpe_uri': 'your-uri-here', + 'affected_package': 'your-package-here', + 'affected_version_start': { + 'kind': Version.VersionKind.MINIMUM + }, + 'fixed_version': { + 'kind': Version.VersionKind.MAXIMUM + } + } + ] + } + } + grafeas_client.\ + create_note(grafeas_client.project_path(PROJECT_ID), note_id, note) + occurrence = { + 'note_name': f"projects/{PROJECT_ID}/notes/{note_id}", + 'resource_uri': self.image_url, + 'vulnerability': { + 'effective_severity': Severity.CRITICAL, + 'package_issue': [ + { + 'affected_cpe_uri': 'your-uri-here', + 'affected_package': 'your-package-here', + 'affected_version': { + 'kind': Version.VersionKind.MINIMUM + }, + 'fixed_version': { + 'kind': Version.VersionKind.MAXIMUM + } + } + ] + } + } + created = grafeas_client.\ + create_occurrence(grafeas_client.project_path(PROJECT_ID), + occurrence) + # query again + tries = 0 + count = 0 + while count != 1 and tries < TRY_LIMIT: + tries += 1 + occ_list = samples.find_vulnerabilities_for_image(self.image_url, + PROJECT_ID) + count = len(occ_list) + time.sleep(SLEEP_TIME) + assert len(occ_list) == 1 + # clean up + samples.delete_occurrence(basename(created.name), PROJECT_ID) + samples.delete_note(note_id, PROJECT_ID) diff --git a/setup.py b/setup.py index b46d135..3899f26 100644 --- a/setup.py +++ b/setup.py @@ -22,13 +22,13 @@ name = "google-cloud-containeranalysis" description = "Container Analysis API API client library" -version = "1.0.2" +version = "1.0.3" release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "google-api-core[grpc] >= 1.14.0, < 2.0.0dev", "grpc-google-iam-v1 >= 0.12.3, < 0.13dev", 'enum34; python_version < "3.4"', - "grafeas", + "grafeas < 1.0.0dev", ] diff --git a/synth.metadata b/synth.metadata index da2fe9f..dcb8b0f 100644 --- a/synth.metadata +++ b/synth.metadata @@ -3,23 +3,30 @@ { "git": { "name": ".", - "remote": "https://github.com/googleapis/python-containeranalysis.git", - "sha": "ddf95852778c0e60961516ecd77b793e6af3295b" + "remote": "git@github.com:busunkim96/python-containeranalysis.git", + "sha": "d8ee9edbfa95d596d76d1de5770dc00e094c8fc7" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "db69b46790b55a82ab7cfa473d031da787bc7591", - "internalRef": "320411362" + "sha": "fb84629a56703d04f0b5304c4a9ade7313ebd92d", + "internalRef": "325339219" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "303271797a360f8a439203413f13a160f2f5b3b4" + "sha": "5f2f711c91199ba2f609d3f06a2fe22aee4e5be3" + } + }, + { + "git": { + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "5f2f711c91199ba2f609d3f06a2fe22aee4e5be3" } } ], diff --git a/synth.py b/synth.py index b65d81d..c2edaab 100644 --- a/synth.py +++ b/synth.py @@ -17,6 +17,7 @@ import synthtool as s import synthtool.gcp as gcp import logging +from synthtool.languages import python logging.basicConfig(level=logging.DEBUG) @@ -85,7 +86,10 @@ def set_iam_policy(''', templated_files = common.py_library(unit_cov_level=45, cov_level=45) s.move(templated_files) +python.py_samples(skip_readmes=True) + # TODO(busunkim): Use latest sphinx after microgenerator transition s.replace("noxfile.py", """['"]sphinx['"]""", '"sphinx<3.0.0"') + s.shell.run(["nox", "-s", "blacken"], hide_output=False)