diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml new file mode 100644 index 00000000..33581568 --- /dev/null +++ b/.github/blunderbuss.yml @@ -0,0 +1,9 @@ +assign_prs: + - KaylaNguyen + - HKWinterhalter + - janell-chen + +assign_issues: + - KaylaNguyen + - HKWinterhalter + - janell-chen diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..32ac90d6 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["group:allNonMajor", "schedule:monthly"], + "packageRules": [ + { + "description": "Create a PR whenever there is a new major version", + "matchUpdateTypes": [ + "major" + ] + } + ], + "ignorePaths": [ + "examples/**" + ] +} diff --git a/.github/workflows/buildpack-integration-test.yml b/.github/workflows/buildpack-integration-test.yml index ba4aecb7..78c56996 100644 --- a/.github/workflows/buildpack-integration-test.yml +++ b/.github/workflows/buildpack-integration-test.yml @@ -5,6 +5,10 @@ on: branches: - master workflow_dispatch: + +# Declare default permissions as read only. +permissions: read-all + jobs: python37: uses: GoogleCloudPlatform/functions-framework-conformance/.github/workflows/buildpack-integration-test.yml@v1.8.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..02405c9a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,78 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: ["master"] + pull_request: + # The branches below must be a subset of the branches above + branches: ["master"] + schedule: + - cron: "0 0 * * 1" + +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ["python"] + # CodeQL supports [ $supported-codeql-languages ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: Checkout repository + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # v2.3.2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # v2.3.2 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # v2.3.2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 03f27450..7c9661c6 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,30 +1,39 @@ name: Python Conformance CI on: [push, pull_request] + +# Declare default permissions as read only. +permissions: read-all + jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python }} - name: Install the framework run: python -m pip install -e . - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version: '1.16' - name: Run HTTP conformance tests - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'http' @@ -33,7 +42,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_http --signature-type http'" - name: Run event conformance tests - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'legacyevent' @@ -42,7 +51,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_legacy_event --signature-type event'" - name: Run CloudEvents conformance tests - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'cloudevent' @@ -51,7 +60,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_cloud_event --signature-type cloudevent'" - name: Run HTTP conformance tests declarative - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'http' @@ -60,7 +69,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_http_declarative'" - name: Run CloudEvents conformance tests declarative - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'cloudevent' @@ -69,7 +78,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_cloud_event_declarative'" - name: Run HTTP concurrency tests declarative - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'http' @@ -78,7 +87,7 @@ jobs: cmd: "'functions-framework --source tests/conformance/main.py --target write_http_declarative_concurrent'" - name: Run Typed tests declarative - uses: GoogleCloudPlatform/functions-framework-conformance/action@v1.6.0 + uses: GoogleCloudPlatform/functions-framework-conformance/action@1975792fb34ebbfa058d690666186d669d3a5977 # v1.8.0 with: version: 'v1.6.0' functionType: 'http' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000..369f9f99 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,27 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, +# surfacing known-vulnerable versions of the packages declared or updated in the PR. +# Once installed, if the workflow run is marked as required, +# PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: 'Checkout Repository' + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - name: 'Dependency Review' + uses: actions/dependency-review-action@f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e # v3.0.4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7bb0972e..c8df3207 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,12 +1,20 @@ name: Python Lint CI on: [push, pull_request] +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1 - name: Install tox run: python -m pip install tox - name: Lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31d80732..6c151db3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,23 +4,31 @@ on: release: types: [published] +permissions: + contents: read + jobs: build-and-pubish: name: Build and Publish runs-on: ubuntu-latest steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: ref: ${{ github.event.release.tag_name }} - name: Install Python - uses: actions/setup-python@v2 + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1 - name: Install build dependencies run: python -m pip install -U setuptools build wheel - name: Build distributions run: python -m build - name: Publish - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@a56da0b891b3dc519c7ee3284aff1fad93cc8598 # master with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..60b0f355 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,52 @@ +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '0 */12 * * *' + push: + branches: [ "master" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + + steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: "Checkout code" + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@80e868c13c90f172d68d1f4501dee99e2479f7af # v2.1.3 + with: + results_file: results.sarif + results_format: sarif + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # v2.3.2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 39bd1be1..68c19e6d 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -1,5 +1,8 @@ name: Python Unit CI on: [push, pull_request] +permissions: + contents: read + jobs: test: strategy: @@ -8,10 +11,15 @@ jobs: platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: + - name: Harden Runner + uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1 with: python-version: ${{ matrix.python }} - name: Install tox diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e83453..9016fc49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.4.0](https://github.com/GoogleCloudPlatform/functions-framework-python/compare/v3.3.0...v3.4.0) (2023-05-24) + + +### Features + +* configure security score card action ([#216](https://github.com/GoogleCloudPlatform/functions-framework-python/issues/216)) ([7868dc1](https://github.com/GoogleCloudPlatform/functions-framework-python/commit/7868dc110c048d3e1acf082faf36b75c3770e3f3)) + + +### Bug Fixes + +* streaming requests cannot access request data ([#245](https://github.com/GoogleCloudPlatform/functions-framework-python/issues/245)) ([c492b04](https://github.com/GoogleCloudPlatform/functions-framework-python/commit/c492b04e87a55194b7709e471b0ec3e2c630f288)) + ## [3.3.0](https://github.com/GoogleCloudPlatform/functions-framework-python/compare/v3.2.1...v3.3.0) (2022-12-16) diff --git a/examples/cloud_run_cloud_events/Dockerfile b/examples/cloud_run_cloud_events/Dockerfile index bc9df896..1bae67aa 100644 --- a/examples/cloud_run_cloud_events/Dockerfile +++ b/examples/cloud_run_cloud_events/Dockerfile @@ -1,6 +1,6 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.7-slim +FROM python:3.7-slim@sha256:adbcdfcd0511bab2d6db252e55b983da1b431598ed755c1620b291fbeb5f6f72 # Copy local code to the container image. ENV APP_HOME /app diff --git a/examples/cloud_run_decorator/Dockerfile b/examples/cloud_run_decorator/Dockerfile index 717e5a91..cc3f44c8 100644 --- a/examples/cloud_run_decorator/Dockerfile +++ b/examples/cloud_run_decorator/Dockerfile @@ -1,6 +1,6 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.7-slim +FROM python:3.7-slim@sha256:adbcdfcd0511bab2d6db252e55b983da1b431598ed755c1620b291fbeb5f6f72 # Copy local code to the container image. ENV APP_HOME /app diff --git a/examples/cloud_run_event/Dockerfile b/examples/cloud_run_event/Dockerfile index 7fa0df13..d3b4c571 100644 --- a/examples/cloud_run_event/Dockerfile +++ b/examples/cloud_run_event/Dockerfile @@ -1,6 +1,6 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.7-slim +FROM python:3.7-slim@sha256:adbcdfcd0511bab2d6db252e55b983da1b431598ed755c1620b291fbeb5f6f72 # Copy local code to the container image. ENV APP_HOME /app diff --git a/examples/cloud_run_http/Dockerfile b/examples/cloud_run_http/Dockerfile index b7d6f502..14f2b2e4 100644 --- a/examples/cloud_run_http/Dockerfile +++ b/examples/cloud_run_http/Dockerfile @@ -1,6 +1,6 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.7-slim +FROM python:3.7-slim@sha256:adbcdfcd0511bab2d6db252e55b983da1b431598ed755c1620b291fbeb5f6f72 # Copy local code to the container image. ENV APP_HOME /app diff --git a/setup.py b/setup.py index 46b4bdfe..9e8aedab 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( name="functions-framework", - version="3.3.0", + version="3.4.0", description="An open source FaaS (Function as a service) framework for writing portable Python functions -- brought to you by the Google Cloud Functions team.", long_description=long_description, long_description_content_type="text/markdown", diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index c2a52d74..d4575b57 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -294,10 +294,13 @@ def _configure_app(app, function, signature_type): def read_request(response): """ Force the framework to read the entire request before responding, to avoid - connection errors when returning prematurely. + connection errors when returning prematurely. Skipped on streaming responses + as these may continue to operate on the request after they are returned. """ - flask.request.get_data() + if not response.is_streamed: + flask.request.get_data() + return response diff --git a/tests/test_cloud_event_functions.py b/tests/test_cloud_event_functions.py index a673c6e4..691fe388 100644 --- a/tests/test_cloud_event_functions.py +++ b/tests/test_cloud_event_functions.py @@ -187,7 +187,8 @@ def test_invalid_fields_binary(client, create_headers_binary, data_payload): def test_unparsable_cloud_event(client): - resp = client.post("/", headers={}, data="") + headers = {"Content-Type": "application/cloudevents+json"} + resp = client.post("/", headers=headers, data="") assert resp.status_code == 400 assert "Bad Request" in resp.data.decode() diff --git a/tests/test_functions.py b/tests/test_functions.py index c26cb625..81860cae 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import io import json import pathlib import re @@ -251,7 +251,8 @@ def test_pubsub_payload(background_event_client, background_json): def test_background_function_no_data(background_event_client, background_json): - resp = background_event_client.post("/") + headers = {"Content-Type": "application/json"} + resp = background_event_client.post("/", headers=headers) assert resp.status_code == 400 @@ -489,6 +490,18 @@ def test_function_returns_none(): assert resp.status_code == 500 +def test_function_returns_stream(): + source = TEST_FUNCTIONS_DIR / "http_streaming" / "main.py" + target = "function" + + client = create_app(target, source).test_client() + resp = client.post("/", data="1\n2\n3\n4\n") + + assert resp.status_code == 200 + assert resp.is_streamed + assert resp.data.decode("utf-8") == "1.0\n3.0\n6.0\n10.0\n" + + def test_legacy_function_check_env(monkeypatch): source = TEST_FUNCTIONS_DIR / "http_check_env" / "main.py" target = "function" diff --git a/tests/test_functions/http_streaming/main.py b/tests/test_functions/http_streaming/main.py new file mode 100644 index 00000000..4b249697 --- /dev/null +++ b/tests/test_functions/http_streaming/main.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Function used in Worker tests of handling HTTP functions.""" + +import flask + +from flask import Response, stream_with_context + + +def function(request): + """Test HTTP function that reads a stream of integers and returns a stream + providing the sum of values read so far. + + Args: + request: The HTTP request which triggered this function. Must contain a + stream of new line separated integers. + + Returns: + Value and status code defined for the given mode. + + Raises: + Exception: Thrown when requested in the incoming mode specification. + """ + print("INVOKED THE STREAM FUNCTION!!!") + + def generate(): + sum_so_far = 0 + for line in request.stream: + sum_so_far += float(line) + yield (str(sum_so_far) + "\n").encode("utf-8") + + return Response(stream_with_context(generate())) diff --git a/tox.ini b/tox.ini index fb76a0e8..0fe3dba6 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = py{35,36,37,38,39,310}-{ubuntu-latest,macos-latest,windows-latest},lin [testenv] usedevelop = true deps = - docker<5 # https://github.com/docker/docker-py/issues/2807 + docker pytest-cov pytest-integration pretend