diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..c0a95e72 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [ + "optional-kubernetes-engine" + ] +} diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1 @@ + diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 00000000..7d8eca8e --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,40 @@ +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: true + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: false + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: true + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "kokoro" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: + # Team slug to add to repository permissions + - team: yoshi-admins + # Access level required, one of push|pull|admin + permission: admin + - team: python-samples-reviewers + permission: admin diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg index c55b58f2..f58e4f76 100644 --- a/.kokoro/common.cfg +++ b/.kokoro/common.cfg @@ -7,16 +7,22 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/getting-started-python" # All builds use the trampoline script to run in docker. -build_file: "getting-started-python/.kokoro/trampoline.sh" +build_file: "getting-started-python/.kokoro/trampoline_v2.sh" # Use the Python worker docker iamge. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:4b6ba8c199e96248980db4538065cddeea594138b9b9fb2d0388603922087747" + value: "gcr.io/cloud-devrel-kokoro-resources/python/getting-started-python" } # Tell the trampoline which build file to use. env_vars: { key: "TRAMPOLINE_BUILD_FILE" - value: "github/getting-started-python/.kokoro/system_tests.sh" + value: ".kokoro/system_tests.sh" +} + +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" } diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile new file mode 100644 index 00000000..accdd0bf --- /dev/null +++ b/.kokoro/docker/Dockerfile @@ -0,0 +1,57 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-devrel-kokoro-resources/python-base:latest + +# Install libraries needed by third-party python packages that we depend on. +RUN apt-get update \ + && apt-get install -y \ + graphviz \ + libcurl4-openssl-dev \ + libffi-dev \ + libjpeg-dev \ + libmagickwand-dev \ + libmemcached-dev \ + libmysqlclient-dev \ + libpng-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + openssl \ + zlib1g-dev \ + && apt-get clean + + +###################### Check python version + +RUN python3 --version +RUN which python3 + +# Setup Cloud SDK +ENV CLOUD_SDK_VERSION 489.0.0 +# Use system python for cloud sdk. +ENV CLOUDSDK_PYTHON python3.12 +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN /google-cloud-sdk/install.sh +ENV PATH /google-cloud-sdk/bin:$PATH + +# Setup the user profile for pip +ENV PATH ~/.local/bin:/root/.local/bin:$PATH + +# Install the current version of nox. +RUN python3 -m pip install --user --no-cache-dir nox==2022.1.7 + +CMD ["nox"] diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index f60e50fe..29bb4ae5 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -18,7 +18,8 @@ set -eo pipefail export PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin -cd github/getting-started-python +cd "${PROJECT_ROOT:-github/getting-started-python}" + # Unencrypt and extract secrets SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") @@ -27,13 +28,20 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") # Setup environment variables export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service-account.json" +# This block is executed only with Trampoline V2. +if [[ -n "${TRAMPOLINE_VERSION:-}" ]]; then + # Install nox as a user and add it to the PATH. + python3 -m pip install --user nox + export PATH="${PATH}:${HOME}/.local/bin" +fi + # Run tests nox -s lint nox -s run_tests -# If this is a nightly build, send the test log to the Build Cop Bot. -# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. +# If this is a nightly build, send the test log to the Flaky Bot. +# See https://github.com/googleapis/repo-automation-bots/tree/HEAD/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"system_tests"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop - $KOKORO_GFILE_DIR/linux_amd64/buildcop -fi \ No newline at end of file + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot +fi diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..ef6972b4 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.10" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Flaky Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + "KOKORO_BUILD_ARTIFACTS_SUBDIR" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..17f21195 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + # We test this envvar in tests/python/test_envvar.py. + "TEST_ENV" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# The build will show some commands and docker build logs. +TRAMPOLINE_VERBOSE="true" diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0dfefd93 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The python-samples-owners team is the default owner for anything not +# explicitly taken by someone else. +* @GoogleCloudPlatform/python-samples-reviewers diff --git a/README.md b/README.md index 0e5b3292..a347e9bb 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ The code for the samples is contained in individual folders in this repository. Tutorial | Folder ---------|------- -[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/bookshelf) -[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/background) -[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/master/quickstart/python) -[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/gce) -[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/sessions) -[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/authenticating-users) +[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/bookshelf) +[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/background) +[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/quickstart/python) +[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/gce) +[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/sessions) +[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/authenticating-users) ## Contributing changes diff --git a/authenticating-users/main_test.py b/authenticating-users/main_test.py new file mode 100644 index 00000000..38b934b6 --- /dev/null +++ b/authenticating-users/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def fake_validate(assertion): + if assertion == "Valid": + return "nobody@example.com", "user0001" + else: + return None, None + + +main.validate_assertion = fake_validate + + +def test_home_page(): + client = main.app.test_client() + + # Good request check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Valid"}) + assert "nobody@example.com" in r.text + + # Missing header check + r = client.get("/") + assert "None" in r.text + + # Bad header check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Not Valid"}) + assert "None" in r.text diff --git a/authenticating-users/requirements-test.txt b/authenticating-users/requirements-test.txt new file mode 100644 index 00000000..6a3d7bca --- /dev/null +++ b/authenticating-users/requirements-test.txt @@ -0,0 +1 @@ +pytest==7.1.2 \ No newline at end of file diff --git a/authenticating-users/requirements.txt b/authenticating-users/requirements.txt index b754d36e..0d4a3c36 100644 --- a/authenticating-users/requirements.txt +++ b/authenticating-users/requirements.txt @@ -1,6 +1,6 @@ # [START getting_started_requirements] -Flask==1.1.1 -cryptography==2.8 -python-jose[cryptography]==3.0.1 -requests==2.22.0 +Flask==2.2.5 +cryptography==41.0.2 +python-jose[cryptography]==3.3.0 +requests==2.31.0 # [END getting_started_requirements] diff --git a/background/app/app.yaml b/background/app/app.yaml index fb63451a..02c0651c 100644 --- a/background/app/app.yaml +++ b/background/app/app.yaml @@ -13,5 +13,5 @@ # limitations under the License. # [START getting_started_background_config] -runtime: python37 +runtime: python312 # [END getting_started_background_config] diff --git a/background/app/main.py b/background/app/main.py index 487e10a4..fad93d46 100644 --- a/background/app/main.py +++ b/background/app/main.py @@ -21,8 +21,8 @@ import os from flask import Flask, redirect, render_template, request -from google.cloud import firestore -from google.cloud import pubsub +from google.cloud import firestore, pubsub +from markupsafe import escape app = Flask(__name__) @@ -32,53 +32,57 @@ publisher = pubsub.PublisherClient() # Keep this list of supported languages up to date -ACCEPTABLE_LANGUAGES = ('de', 'en', 'es', 'fr', 'ja', 'sw') +ACCEPTABLE_LANGUAGES = ("de", "en", "es", "fr", "ja", "sw") # [END getting_started_background_app_main] # [START getting_started_background_app_list] -@app.route('/', methods=['GET']) +@app.route("/", methods=["GET"]) def index(): - """ The home page has a list of prior translations and a form to - ask for a new translation. + """The home page has a list of prior translations and a form to + ask for a new translation. """ doc_list = [] - docs = db.collection('translations').stream() + docs = db.collection("translations").stream() for doc in docs: doc_list.append(doc.to_dict()) - return render_template('index.html', translations=doc_list) + return render_template("index.html", translations=doc_list) + + # [END getting_started_background_app_list] # [START getting_started_background_app_request] -@app.route('/request-translation', methods=['POST']) +@app.route("/request-translation", methods=["POST"]) def translate(): - """ Handle a request to translate a string (form field 'v') to a given - language (form field 'lang'), by sending a PubSub message to a topic. + """Handle a request to translate a string (form field 'v') to a given + language (form field 'lang'), by sending a PubSub message to a topic. """ - source_string = request.form.get('v', '') - to_language = request.form.get('lang', '') + source_string = request.form.get("v", "") + to_language = escape(request.form.get("lang", "")) - if source_string == '': - error_message = 'Empty value' - return error_message, 400 + if source_string == "": + return "Invalid request, you must provide a value.", 400 if to_language not in ACCEPTABLE_LANGUAGES: - error_message = 'Unsupported language: {}'.format(to_language) - return error_message, 400 + return f"Unsupported language: {to_language}", 400 message = { - 'Original': source_string, - 'Language': to_language, - 'Translated': '', - 'OriginalLanguage': '', + "Original": source_string, + "Language": to_language, + "Translated": "", + "OriginalLanguage": "", } - topic_name = 'projects/{}/topics/{}'.format( - os.getenv('GOOGLE_CLOUD_PROJECT'), 'translate' + topic_name = ( + f"projects/{os.getenv('GOOGLE_CLOUD_PROJECT')}/topics/translate" + ) + publisher.publish( + topic=topic_name, data=json.dumps(message).encode("utf-8") ) - publisher.publish(topic_name, json.dumps(message).encode('utf8')) - return redirect('/') + return redirect("/") + + # [END getting_started_background_app_request] diff --git a/background/app/main_test.py b/background/app/main_test.py index 0832fb3b..41b529d3 100644 --- a/background/app/main_test.py +++ b/background/app/main_test.py @@ -16,59 +16,61 @@ import uuid import google.auth -from google.cloud import firestore -from google.cloud import pubsub +from google.cloud import firestore, pubsub, storage import main import pytest credentials, project_id = google.auth.default() -os.environ['GOOGLE_CLOUD_PROJECT'] = project_id -SUBSCRIPTION_NAME = 'projects/{}/subscriptions/{}'.format( - project_id, 'test-' + str(uuid.uuid4()) -) -TOPIC_NAME = 'projects/{}/topics/{}'.format( - project_id, 'translate' +os.environ["GOOGLE_CLOUD_PROJECT"] = project_id +SUBSCRIPTION_NAME = "projects/{}/subscriptions/{}".format( + project_id, "test-" + str(uuid.uuid4()) ) +TOPIC_NAME = "projects/{}/topics/{}".format(project_id, "translate") -@pytest.yield_fixture +@pytest.fixture def db(): def clear_collection(collection): - """ Removes every document from the collection, to make it easy to see - what has been added by the current test run. + """Removes every document from the collection, to make it easy to see + what has been added by the current test run. """ for doc in collection.stream(): doc.reference.delete() + bucket_name = 'system-test-bucket' client = firestore.Client() - translations = client.collection('translations') + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + translations = client.collection("translations") clear_collection(translations) - translations.add({ - 'Original': 'A testing message', - 'Language': 'fr', - 'Translated': '"A testing message", but in French', - 'OriginalLanguage': 'en', + translations.add( + { + "Original": "A testing message", + "Language": "fr", + "Translated": '"A testing message", but in French', + "OriginalLanguage": "en", }, - document_id='test translation' + document_id="test translation", ) + assert bucket in locals() yield client -@pytest.yield_fixture +@pytest.fixture def publisher(): client = pubsub.PublisherClient() yield client -@pytest.yield_fixture +@pytest.fixture def subscriber(): subscriber = pubsub.SubscriberClient() subscriber.create_subscription( - SUBSCRIPTION_NAME, TOPIC_NAME + request={"name": SUBSCRIPTION_NAME, "topic": TOPIC_NAME} ) yield subscriber - subscriber.delete_subscription(SUBSCRIPTION_NAME) + subscriber.delete_subscription(request={"subscription": SUBSCRIPTION_NAME}) def test_index(db, publisher): @@ -77,11 +79,11 @@ def test_index(db, publisher): main.publisher = publisher client = main.app.test_client() - r = client.get('/') + r = client.get("/") assert r.status_code == 200 - response_text = r.data.decode('utf-8') - assert 'Text to translate' in response_text - assert 'but in French' in response_text + response_text = r.data.decode("utf-8") + assert "Text to translate" in response_text + assert "but in French" in response_text def test_translate(db, publisher, subscriber): @@ -90,14 +92,20 @@ def test_translate(db, publisher, subscriber): main.publisher = publisher client = main.app.test_client() - r = client.post('/request-translation', data={ - 'v': 'This is a test', - 'lang': 'fr', - }) + r = client.post( + "/request-translation", + data={ + "v": "This is a test", + "lang": "fr", + }, + ) assert r.status_code < 400 - response = subscriber.pull(SUBSCRIPTION_NAME, 1, timeout=10.0) + response = subscriber.pull( + request={"subscription": SUBSCRIPTION_NAME, "max_messages": 1}, + timeout=10.0, + ) assert len(response.received_messages) == 1 - assert b'This is a test' in response.received_messages[0].message.data - assert b'fr' in response.received_messages[0].message.data + assert b"This is a test" in response.received_messages[0].message.data + assert b"fr" in response.received_messages[0].message.data diff --git a/background/app/requirements.txt b/background/app/requirements.txt index a7cd28d6..f70d16b7 100644 --- a/background/app/requirements.txt +++ b/background/app/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-firestore==1.6.0 -google-cloud-pubsub==1.0.2 -flask==1.1.1 +google-cloud-firestore==2.18.0 +google-cloud-pubsub==2.23.0 +flask==3.0.3 diff --git a/background/function/requirements.txt b/background/function/requirements.txt index 2aab9b14..b8e6aaad 100644 --- a/background/function/requirements.txt +++ b/background/function/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-translate==2.0.0 -google-cloud-firestore==1.6.0 +google-cloud-translate==3.11.1 +google-cloud-firestore==2.11.1 diff --git a/bookshelf/Dockerfile b/bookshelf/Dockerfile new file mode 100644 index 00000000..44c98b38 --- /dev/null +++ b/bookshelf/Dockerfile @@ -0,0 +1,14 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.11-slim + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install --no-cache-dir -r requirements.txt + +# Run the web service on container startup. +ENTRYPOINT [ "gunicorn", "--bind", "0.0.0.0:8080", "main:app" ] \ No newline at end of file diff --git a/bookshelf/main.py b/bookshelf/main.py index 1962c485..789bec2e 100644 --- a/bookshelf/main.py +++ b/bookshelf/main.py @@ -59,7 +59,7 @@ def upload_image_file(img): logging.basicConfig(level=logging.INFO) client = google.cloud.logging.Client() # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) + client.setup_logging() @app.route('/') diff --git a/bookshelf/main_test.py b/bookshelf/main_test.py index 404de778..249cba17 100644 --- a/bookshelf/main_test.py +++ b/bookshelf/main_test.py @@ -26,7 +26,7 @@ os.environ['GOOGLE_CLOUD_PROJECT'] = project_id -@pytest.yield_fixture +@pytest.fixture def app(request): """This fixture provides a Flask app instance configured for testing. @@ -38,17 +38,17 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def firestore(): + """This fixture provides a modified version of the app's Firebase model + that tracks all created items and deletes them at the end of the test. - import firestore - """This fixture provides a modified version of the app's Firebase model that - tracks all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. + Any tests that directly or indirectly interact with the database should + use this to ensure that resources are properly cleaned up. """ + import firestore + # Ensure no books exist before running the tests. This typically helps if # tests somehow left the database in a bad state. delete_all_books(firestore) diff --git a/bookshelf/requirements.txt b/bookshelf/requirements.txt index b3e68adc..27d3d4fe 100644 --- a/bookshelf/requirements.txt +++ b/bookshelf/requirements.txt @@ -1,7 +1,7 @@ -Flask>=1.0.0 -google-cloud-firestore==1.6.0 -google-cloud-storage==1.23.0 -google-cloud-logging==1.14.0 -google-cloud-error-reporting==0.33.0 -gunicorn==19.9.0 -six>=1.11.0 +Flask==2.2.5 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.9.0 +google-cloud-error-reporting==1.9.1 +google-cloud-logging==3.5.0 +gunicorn==20.1.0 +six==1.16.0 diff --git a/bookshelf/storage.py b/bookshelf/storage.py index f6f9b1d3..f18b78df 100644 --- a/bookshelf/storage.py +++ b/bookshelf/storage.py @@ -63,6 +63,8 @@ def upload_file(file_stream, filename, content_type): blob.upload_from_string( file_stream, content_type=content_type) + # Ensure the file is publicly readable. + blob.make_public() url = blob.public_url # [END bookshelf_cloud_storage_client] diff --git a/bookshelf/templates/list.html b/bookshelf/templates/list.html index ffbad499..b255d209 100644 --- a/bookshelf/templates/list.html +++ b/bookshelf/templates/list.html @@ -37,7 +37,7 @@