diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..c0a95e72 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [ + "optional-kubernetes-engine" + ] +} diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1 @@ + diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 00000000..7d8eca8e --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,40 @@ +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: true + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: false + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: true + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "kokoro" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: + # Team slug to add to repository permissions + - team: yoshi-admins + # Access level required, one of push|pull|admin + permission: admin + - team: python-samples-reviewers + permission: admin diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg index c55b58f2..f58e4f76 100644 --- a/.kokoro/common.cfg +++ b/.kokoro/common.cfg @@ -7,16 +7,22 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/getting-started-python" # All builds use the trampoline script to run in docker. -build_file: "getting-started-python/.kokoro/trampoline.sh" +build_file: "getting-started-python/.kokoro/trampoline_v2.sh" # Use the Python worker docker iamge. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:4b6ba8c199e96248980db4538065cddeea594138b9b9fb2d0388603922087747" + value: "gcr.io/cloud-devrel-kokoro-resources/python/getting-started-python" } # Tell the trampoline which build file to use. env_vars: { key: "TRAMPOLINE_BUILD_FILE" - value: "github/getting-started-python/.kokoro/system_tests.sh" + value: ".kokoro/system_tests.sh" +} + +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" } diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile new file mode 100644 index 00000000..accdd0bf --- /dev/null +++ b/.kokoro/docker/Dockerfile @@ -0,0 +1,57 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-devrel-kokoro-resources/python-base:latest + +# Install libraries needed by third-party python packages that we depend on. +RUN apt-get update \ + && apt-get install -y \ + graphviz \ + libcurl4-openssl-dev \ + libffi-dev \ + libjpeg-dev \ + libmagickwand-dev \ + libmemcached-dev \ + libmysqlclient-dev \ + libpng-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + openssl \ + zlib1g-dev \ + && apt-get clean + + +###################### Check python version + +RUN python3 --version +RUN which python3 + +# Setup Cloud SDK +ENV CLOUD_SDK_VERSION 489.0.0 +# Use system python for cloud sdk. +ENV CLOUDSDK_PYTHON python3.12 +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN /google-cloud-sdk/install.sh +ENV PATH /google-cloud-sdk/bin:$PATH + +# Setup the user profile for pip +ENV PATH ~/.local/bin:/root/.local/bin:$PATH + +# Install the current version of nox. +RUN python3 -m pip install --user --no-cache-dir nox==2022.1.7 + +CMD ["nox"] diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index f60e50fe..29bb4ae5 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -18,7 +18,8 @@ set -eo pipefail export PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin -cd github/getting-started-python +cd "${PROJECT_ROOT:-github/getting-started-python}" + # Unencrypt and extract secrets SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") @@ -27,13 +28,20 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") # Setup environment variables export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service-account.json" +# This block is executed only with Trampoline V2. +if [[ -n "${TRAMPOLINE_VERSION:-}" ]]; then + # Install nox as a user and add it to the PATH. + python3 -m pip install --user nox + export PATH="${PATH}:${HOME}/.local/bin" +fi + # Run tests nox -s lint nox -s run_tests -# If this is a nightly build, send the test log to the Build Cop Bot. -# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. +# If this is a nightly build, send the test log to the Flaky Bot. +# See https://github.com/googleapis/repo-automation-bots/tree/HEAD/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"system_tests"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop - $KOKORO_GFILE_DIR/linux_amd64/buildcop -fi \ No newline at end of file + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot +fi diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..ef6972b4 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.10" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Flaky Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + "KOKORO_BUILD_ARTIFACTS_SUBDIR" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..17f21195 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + # We test this envvar in tests/python/test_envvar.py. + "TEST_ENV" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# The build will show some commands and docker build logs. +TRAMPOLINE_VERBOSE="true" diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0dfefd93 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The python-samples-owners team is the default owner for anything not +# explicitly taken by someone else. +* @GoogleCloudPlatform/python-samples-reviewers diff --git a/README.md b/README.md index 0e5b3292..a347e9bb 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ The code for the samples is contained in individual folders in this repository. Tutorial | Folder ---------|------- -[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/bookshelf) -[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/background) -[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/master/quickstart/python) -[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/gce) -[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/sessions) -[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/authenticating-users) +[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/bookshelf) +[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/background) +[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/quickstart/python) +[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/gce) +[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/sessions) +[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/authenticating-users) ## Contributing changes diff --git a/authenticating-users/main_test.py b/authenticating-users/main_test.py new file mode 100644 index 00000000..38b934b6 --- /dev/null +++ b/authenticating-users/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def fake_validate(assertion): + if assertion == "Valid": + return "nobody@example.com", "user0001" + else: + return None, None + + +main.validate_assertion = fake_validate + + +def test_home_page(): + client = main.app.test_client() + + # Good request check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Valid"}) + assert "nobody@example.com" in r.text + + # Missing header check + r = client.get("/") + assert "None" in r.text + + # Bad header check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Not Valid"}) + assert "None" in r.text diff --git a/authenticating-users/requirements-test.txt b/authenticating-users/requirements-test.txt new file mode 100644 index 00000000..6a3d7bca --- /dev/null +++ b/authenticating-users/requirements-test.txt @@ -0,0 +1 @@ +pytest==7.1.2 \ No newline at end of file diff --git a/authenticating-users/requirements.txt b/authenticating-users/requirements.txt index b754d36e..0d4a3c36 100644 --- a/authenticating-users/requirements.txt +++ b/authenticating-users/requirements.txt @@ -1,6 +1,6 @@ # [START getting_started_requirements] -Flask==1.1.1 -cryptography==2.8 -python-jose[cryptography]==3.0.1 -requests==2.22.0 +Flask==2.2.5 +cryptography==41.0.2 +python-jose[cryptography]==3.3.0 +requests==2.31.0 # [END getting_started_requirements] diff --git a/background/app/app.yaml b/background/app/app.yaml index fb63451a..02c0651c 100644 --- a/background/app/app.yaml +++ b/background/app/app.yaml @@ -13,5 +13,5 @@ # limitations under the License. # [START getting_started_background_config] -runtime: python37 +runtime: python312 # [END getting_started_background_config] diff --git a/background/app/main.py b/background/app/main.py index 487e10a4..fad93d46 100644 --- a/background/app/main.py +++ b/background/app/main.py @@ -21,8 +21,8 @@ import os from flask import Flask, redirect, render_template, request -from google.cloud import firestore -from google.cloud import pubsub +from google.cloud import firestore, pubsub +from markupsafe import escape app = Flask(__name__) @@ -32,53 +32,57 @@ publisher = pubsub.PublisherClient() # Keep this list of supported languages up to date -ACCEPTABLE_LANGUAGES = ('de', 'en', 'es', 'fr', 'ja', 'sw') +ACCEPTABLE_LANGUAGES = ("de", "en", "es", "fr", "ja", "sw") # [END getting_started_background_app_main] # [START getting_started_background_app_list] -@app.route('/', methods=['GET']) +@app.route("/", methods=["GET"]) def index(): - """ The home page has a list of prior translations and a form to - ask for a new translation. + """The home page has a list of prior translations and a form to + ask for a new translation. """ doc_list = [] - docs = db.collection('translations').stream() + docs = db.collection("translations").stream() for doc in docs: doc_list.append(doc.to_dict()) - return render_template('index.html', translations=doc_list) + return render_template("index.html", translations=doc_list) + + # [END getting_started_background_app_list] # [START getting_started_background_app_request] -@app.route('/request-translation', methods=['POST']) +@app.route("/request-translation", methods=["POST"]) def translate(): - """ Handle a request to translate a string (form field 'v') to a given - language (form field 'lang'), by sending a PubSub message to a topic. + """Handle a request to translate a string (form field 'v') to a given + language (form field 'lang'), by sending a PubSub message to a topic. """ - source_string = request.form.get('v', '') - to_language = request.form.get('lang', '') + source_string = request.form.get("v", "") + to_language = escape(request.form.get("lang", "")) - if source_string == '': - error_message = 'Empty value' - return error_message, 400 + if source_string == "": + return "Invalid request, you must provide a value.", 400 if to_language not in ACCEPTABLE_LANGUAGES: - error_message = 'Unsupported language: {}'.format(to_language) - return error_message, 400 + return f"Unsupported language: {to_language}", 400 message = { - 'Original': source_string, - 'Language': to_language, - 'Translated': '', - 'OriginalLanguage': '', + "Original": source_string, + "Language": to_language, + "Translated": "", + "OriginalLanguage": "", } - topic_name = 'projects/{}/topics/{}'.format( - os.getenv('GOOGLE_CLOUD_PROJECT'), 'translate' + topic_name = ( + f"projects/{os.getenv('GOOGLE_CLOUD_PROJECT')}/topics/translate" + ) + publisher.publish( + topic=topic_name, data=json.dumps(message).encode("utf-8") ) - publisher.publish(topic_name, json.dumps(message).encode('utf8')) - return redirect('/') + return redirect("/") + + # [END getting_started_background_app_request] diff --git a/background/app/main_test.py b/background/app/main_test.py index 0832fb3b..41b529d3 100644 --- a/background/app/main_test.py +++ b/background/app/main_test.py @@ -16,59 +16,61 @@ import uuid import google.auth -from google.cloud import firestore -from google.cloud import pubsub +from google.cloud import firestore, pubsub, storage import main import pytest credentials, project_id = google.auth.default() -os.environ['GOOGLE_CLOUD_PROJECT'] = project_id -SUBSCRIPTION_NAME = 'projects/{}/subscriptions/{}'.format( - project_id, 'test-' + str(uuid.uuid4()) -) -TOPIC_NAME = 'projects/{}/topics/{}'.format( - project_id, 'translate' +os.environ["GOOGLE_CLOUD_PROJECT"] = project_id +SUBSCRIPTION_NAME = "projects/{}/subscriptions/{}".format( + project_id, "test-" + str(uuid.uuid4()) ) +TOPIC_NAME = "projects/{}/topics/{}".format(project_id, "translate") -@pytest.yield_fixture +@pytest.fixture def db(): def clear_collection(collection): - """ Removes every document from the collection, to make it easy to see - what has been added by the current test run. + """Removes every document from the collection, to make it easy to see + what has been added by the current test run. """ for doc in collection.stream(): doc.reference.delete() + bucket_name = 'system-test-bucket' client = firestore.Client() - translations = client.collection('translations') + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + translations = client.collection("translations") clear_collection(translations) - translations.add({ - 'Original': 'A testing message', - 'Language': 'fr', - 'Translated': '"A testing message", but in French', - 'OriginalLanguage': 'en', + translations.add( + { + "Original": "A testing message", + "Language": "fr", + "Translated": '"A testing message", but in French', + "OriginalLanguage": "en", }, - document_id='test translation' + document_id="test translation", ) + assert bucket in locals() yield client -@pytest.yield_fixture +@pytest.fixture def publisher(): client = pubsub.PublisherClient() yield client -@pytest.yield_fixture +@pytest.fixture def subscriber(): subscriber = pubsub.SubscriberClient() subscriber.create_subscription( - SUBSCRIPTION_NAME, TOPIC_NAME + request={"name": SUBSCRIPTION_NAME, "topic": TOPIC_NAME} ) yield subscriber - subscriber.delete_subscription(SUBSCRIPTION_NAME) + subscriber.delete_subscription(request={"subscription": SUBSCRIPTION_NAME}) def test_index(db, publisher): @@ -77,11 +79,11 @@ def test_index(db, publisher): main.publisher = publisher client = main.app.test_client() - r = client.get('/') + r = client.get("/") assert r.status_code == 200 - response_text = r.data.decode('utf-8') - assert 'Text to translate' in response_text - assert 'but in French' in response_text + response_text = r.data.decode("utf-8") + assert "Text to translate" in response_text + assert "but in French" in response_text def test_translate(db, publisher, subscriber): @@ -90,14 +92,20 @@ def test_translate(db, publisher, subscriber): main.publisher = publisher client = main.app.test_client() - r = client.post('/request-translation', data={ - 'v': 'This is a test', - 'lang': 'fr', - }) + r = client.post( + "/request-translation", + data={ + "v": "This is a test", + "lang": "fr", + }, + ) assert r.status_code < 400 - response = subscriber.pull(SUBSCRIPTION_NAME, 1, timeout=10.0) + response = subscriber.pull( + request={"subscription": SUBSCRIPTION_NAME, "max_messages": 1}, + timeout=10.0, + ) assert len(response.received_messages) == 1 - assert b'This is a test' in response.received_messages[0].message.data - assert b'fr' in response.received_messages[0].message.data + assert b"This is a test" in response.received_messages[0].message.data + assert b"fr" in response.received_messages[0].message.data diff --git a/background/app/requirements.txt b/background/app/requirements.txt index a7cd28d6..f70d16b7 100644 --- a/background/app/requirements.txt +++ b/background/app/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-firestore==1.6.0 -google-cloud-pubsub==1.0.2 -flask==1.1.1 +google-cloud-firestore==2.18.0 +google-cloud-pubsub==2.23.0 +flask==3.0.3 diff --git a/background/function/requirements.txt b/background/function/requirements.txt index 2aab9b14..b8e6aaad 100644 --- a/background/function/requirements.txt +++ b/background/function/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-translate==2.0.0 -google-cloud-firestore==1.6.0 +google-cloud-translate==3.11.1 +google-cloud-firestore==2.11.1 diff --git a/bookshelf/Dockerfile b/bookshelf/Dockerfile new file mode 100644 index 00000000..44c98b38 --- /dev/null +++ b/bookshelf/Dockerfile @@ -0,0 +1,14 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.11-slim + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install --no-cache-dir -r requirements.txt + +# Run the web service on container startup. +ENTRYPOINT [ "gunicorn", "--bind", "0.0.0.0:8080", "main:app" ] \ No newline at end of file diff --git a/bookshelf/main.py b/bookshelf/main.py index 1962c485..789bec2e 100644 --- a/bookshelf/main.py +++ b/bookshelf/main.py @@ -59,7 +59,7 @@ def upload_image_file(img): logging.basicConfig(level=logging.INFO) client = google.cloud.logging.Client() # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) + client.setup_logging() @app.route('/') diff --git a/bookshelf/main_test.py b/bookshelf/main_test.py index 404de778..249cba17 100644 --- a/bookshelf/main_test.py +++ b/bookshelf/main_test.py @@ -26,7 +26,7 @@ os.environ['GOOGLE_CLOUD_PROJECT'] = project_id -@pytest.yield_fixture +@pytest.fixture def app(request): """This fixture provides a Flask app instance configured for testing. @@ -38,17 +38,17 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def firestore(): + """This fixture provides a modified version of the app's Firebase model + that tracks all created items and deletes them at the end of the test. - import firestore - """This fixture provides a modified version of the app's Firebase model that - tracks all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. + Any tests that directly or indirectly interact with the database should + use this to ensure that resources are properly cleaned up. """ + import firestore + # Ensure no books exist before running the tests. This typically helps if # tests somehow left the database in a bad state. delete_all_books(firestore) diff --git a/bookshelf/requirements.txt b/bookshelf/requirements.txt index b3e68adc..27d3d4fe 100644 --- a/bookshelf/requirements.txt +++ b/bookshelf/requirements.txt @@ -1,7 +1,7 @@ -Flask>=1.0.0 -google-cloud-firestore==1.6.0 -google-cloud-storage==1.23.0 -google-cloud-logging==1.14.0 -google-cloud-error-reporting==0.33.0 -gunicorn==19.9.0 -six>=1.11.0 +Flask==2.2.5 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.9.0 +google-cloud-error-reporting==1.9.1 +google-cloud-logging==3.5.0 +gunicorn==20.1.0 +six==1.16.0 diff --git a/bookshelf/storage.py b/bookshelf/storage.py index f6f9b1d3..f18b78df 100644 --- a/bookshelf/storage.py +++ b/bookshelf/storage.py @@ -63,6 +63,8 @@ def upload_file(file_stream, filename, content_type): blob.upload_from_string( file_stream, content_type=content_type) + # Ensure the file is publicly readable. + blob.make_public() url = blob.public_url # [END bookshelf_cloud_storage_client] diff --git a/bookshelf/templates/list.html b/bookshelf/templates/list.html index ffbad499..b255d209 100644 --- a/bookshelf/templates/list.html +++ b/bookshelf/templates/list.html @@ -37,7 +37,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/bookshelf/templates/view.html b/bookshelf/templates/view.html index 70ac5a49..476db111 100644 --- a/bookshelf/templates/view.html +++ b/bookshelf/templates/view.html @@ -37,7 +37,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
{# [END book_image] #} diff --git a/decrypt-secrets.sh b/decrypt-secrets.sh index 991919ce..7df5428a 100755 --- a/decrypt-secrets.sh +++ b/decrypt-secrets.sh @@ -14,7 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -password=$1 +set -euo pipefail -openssl aes-256-cbc -k "$password" -in secrets.tar.enc -out secrets.tar -d -tar xvf secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +# If there's already a secret file, skip retrieving the secret. +if [[ -f "service-account.json" ]]; then + echo "The secret already exists, skipping." + exit 0 +fi + +gcloud secrets versions access latest \ + --secret="getting-started-python-service-account" \ + --project="${project_id}" \ + > service-account.json diff --git a/encrypt-secrets.sh b/encrypt-secrets.sh index 5faf4a41..ddad76db 100755 --- a/encrypt-secrets.sh +++ b/encrypt-secrets.sh @@ -14,9 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -read -s -p "Enter password for encryption: " password -echo +set -euo pipefail -tar cvf secrets.tar service-account.json -openssl aes-256-cbc -k "$password" -in secrets.tar -out secrets.tar.enc -rm secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions add "getting-started-python-service-account" \ + --project="${project_id}" \ + --data-file="service-account.json" diff --git a/gce/add-google-cloud-ops-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh new file mode 100644 index 00000000..63cf2dbd --- /dev/null +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -0,0 +1,519 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *NOTE*: The source of truth for this script is: +# https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +# See https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/installation +# for installation instructions. +# It is committed to this repository to follow security best practices. +# +# +# Add repository for the Google ops agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--uninstall-standalone-logging-agent`: +# Uninstalls the standalone logging agent (`google-fluentd`). +# +# * `--uninstall-standalone-monitoring-agent`: +# Uninstalls the standalone monitoring agent (`stackdriver-agent`). +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-google-cloud-ops-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --dry-run +# +# * To replace standalone agents with the Ops agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --uninstall-standalone-logging-agent --uninstall-standalone-monitoring-agent +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-ops-agent-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + uninstall-standalone-logging-agent) ACTIONS+=('uninstall-standalone-logging-agent') ;; + uninstall-standalone-monitoring-agent) ACTIONS+=('uninstall-standalone-monitoring-agent') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ " ${ACTIONS[*]} " == *\ uninstall\ * || ( " ${ACTIONS[*]} " == *\ remove-repo\ * && " ${ACTIONS[*]} " != *\ also-install\ * )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE + +if [[ " ${ACTIONS[*]} " == *\ also-install*uninstall\ * ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the ops agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/stackdriver/docs/solutions/ops-agent' + +# URL documentation which lists supported platforms for running the ops agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#supported_operating_systems" + +# Packages to install. +AGENT_PACKAGE='google-cloud-ops-agent' +declare -a ADDITIONAL_PACKAGES=() + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-ops-agent ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-ops-agent.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-ops-agent.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-ops-agent.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-ops-agent.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 2.0.1~debian9.13. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}~*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early unless at least one package is installed. + dpkg -l "${packages[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-ops-agent.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-ops-agent/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-ops-agent || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +save_configuration_files() { + local save_dir="/var/lib/google-cloud-ops-agent/saved_configs" + ${DRY_RUN} mkdir -p "${save_dir}" + ${DRY_RUN} cp -rp "$@" "${save_dir}" + echo "$* is now copied over to ${save_dir} folder." +} + +main() { + case "${ID:-}" in + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-logging-agent\ * ]]; then + save_configuration_files "/etc/google-fluentd" + # This will also remove dependent packages, e.g. "google-fluentd-catch-all-config" or "google-fluentd-catch-all-config-structured". + uninstall "google-fluentd" + fi + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-monitoring-agent\ * ]]; then + save_configuration_files "/etc/stackdriver" "/opt/stackdriver/collectd/etc" + uninstall "stackdriver-agent" + fi + if [[ " ${ACTIONS[*]} " == *\ add-repo\ * ]]; then + resolve_version + add_repo + fi + if [[ " ${ACTIONS[*]} " == *\ also-install\ * ]]; then + install_agent + elif [[ " ${ACTIONS[*]} " == *\ uninstall\ * ]]; then + save_configuration_files "/etc/google-cloud-ops-agent" + uninstall "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" + fi + if [[ " ${ACTIONS[*]} " == *\ remove-repo\ * ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/deploy.sh b/gce/deploy.sh index b3449837..36ed75ca 100644 --- a/gce/deploy.sh +++ b/gce/deploy.sh @@ -19,7 +19,7 @@ MY_INSTANCE_NAME="my-app-instance" ZONE=us-central1-a gcloud compute instances create $MY_INSTANCE_NAME \ - --image-family=debian-9 \ + --image-family=debian-10 \ --image-project=debian-cloud \ --machine-type=g1-small \ --scopes userinfo-email,cloud-platform \ diff --git a/gce/main_test.py b/gce/main_test.py new file mode 100644 index 00000000..6d1f887b --- /dev/null +++ b/gce/main_test.py @@ -0,0 +1,25 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_hello(): + main.app.testing = True + client = main.app.test_client() + + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Hello, world!" in response_text diff --git a/gce/requirements.txt b/gce/requirements.txt index f561b3c0..b655465f 100644 --- a/gce/requirements.txt +++ b/gce/requirements.txt @@ -1,3 +1,3 @@ -flask==1.1.1 -honcho==1.0.1 -gunicorn==19.9.0 +flask==2.2.5 +honcho==1.1.0 +gunicorn==20.1.0 diff --git a/gce/startup-script.sh b/gce/startup-script.sh index 2068606e..cc35d30f 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -12,29 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Echo commands -set -v +# Echo commands and fail on error +set -ev # [START getting_started_gce_startup_script] -# Install Stackdriver logging agent -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -sudo bash install-logging-agent.sh - # Install or update needed software apt-get update -apt-get install -yq git supervisor python python-pip +apt-get install -yq git supervisor python python-pip python3-distutils pip install --upgrade pip virtualenv -# Account to own server process -useradd -m -d /home/pythonapp pythonapp - # Fetch source code export HOME=/root git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app +# Install Cloud Ops Agent +sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install + +# Account to own server process +useradd -m -d /home/pythonapp pythonapp + # Python environment setup virtualenv -p python3 /opt/app/gce/env -source /opt/app/gce/env/bin/activate +/bin/bash -c "source /opt/app/gce/env/bin/activate" /opt/app/gce/env/bin/pip install -r /opt/app/gce/requirements.txt # Set ownership to newly created account diff --git a/gce/teardown.sh b/gce/teardown.sh index b31d1139..668b1218 100644 --- a/gce/teardown.sh +++ b/gce/teardown.sh @@ -16,8 +16,8 @@ set -x -MY_INSTANCE_NAME="my-instance-name" -ZONE=us-central1-f +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a gcloud compute instances delete $MY_INSTANCE_NAME \ --zone=$ZONE --delete-disks=all diff --git a/noxfile.py b/noxfile.py index 8d7c7d5e..8f5a921c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -55,7 +55,7 @@ def run_test(session, dir): success_codes=[0, 5]) -@nox.session +@nox.session(python="3.12") @nox.parametrize('dir', DIRS) def run_tests(session, dir=None): """Run all tests for all directories (slow!)""" diff --git a/optional-kubernetes-engine/.dockerignore b/optional-kubernetes-engine/.dockerignore index fdc976d4..c4730648 100644 --- a/optional-kubernetes-engine/.dockerignore +++ b/optional-kubernetes-engine/.dockerignore @@ -12,6 +12,9 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log .git +.mypy_cache +.pytest_cache +.hypothesis diff --git a/optional-kubernetes-engine/Makefile b/optional-kubernetes-engine/Makefile index f275ddf4..4c1cbf3e 100644 --- a/optional-kubernetes-engine/Makefile +++ b/optional-kubernetes-engine/Makefile @@ -12,8 +12,8 @@ create-cluster: .PHONY: create-bucket create-bucket: - gsutil mb gs://$(GCLOUD_PROJECT) - gsutil defacl set public-read gs://$(GCLOUD_PROJECT) + gcloud storage buckets create gs://$(GCLOUD_PROJECT) + gcloud storage buckets update gs://$(GCLOUD_PROJECT) --predefined-default-object-acl=public-read .PHONY: build build: diff --git a/optional-kubernetes-engine/README.md b/optional-kubernetes-engine/README.md index e3c17236..9acb98b4 100644 --- a/optional-kubernetes-engine/README.md +++ b/optional-kubernetes-engine/README.md @@ -36,8 +36,8 @@ Alternatively, you can use make: The bookshelf application uses [Google Cloud Storage](https://cloud.google.com/storage) to store image files. Create a bucket for your project: - gsutil mb gs:// - gsutil defacl set public-read gs:// + gcloud storage buckets create gs:// + gcloud storage buckets update gs:// --predefined-default-object-acl=public-read Alternatively, you can use make: diff --git a/optional-kubernetes-engine/bookshelf-frontend.yaml b/optional-kubernetes-engine/bookshelf-frontend.yaml index 265c8e35..5db2d638 100644 --- a/optional-kubernetes-engine/bookshelf-frontend.yaml +++ b/optional-kubernetes-engine/bookshelf-frontend.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf application frontend. The frontend serves # public web traffic. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-frontend @@ -27,6 +27,10 @@ metadata: # https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 3 + selector: + matchLabels: + app: bookshelf + tier: frontend template: metadata: labels: diff --git a/optional-kubernetes-engine/bookshelf-worker.yaml b/optional-kubernetes-engine/bookshelf-worker.yaml index c30df89b..979437b6 100644 --- a/optional-kubernetes-engine/bookshelf-worker.yaml +++ b/optional-kubernetes-engine/bookshelf-worker.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf task worker. The worker is responsible # for processing book requests and updating book information. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-worker @@ -27,6 +27,10 @@ metadata: # https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 2 + selector: + matchLabels: + app: bookshelf + tier: worker template: metadata: labels: diff --git a/optional-kubernetes-engine/bookshelf/templates/list.html b/optional-kubernetes-engine/bookshelf/templates/list.html index 3362f0e2..a80d5b56 100644 --- a/optional-kubernetes-engine/bookshelf/templates/list.html +++ b/optional-kubernetes-engine/bookshelf/templates/list.html @@ -31,7 +31,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/optional-kubernetes-engine/bookshelf/templates/view.html b/optional-kubernetes-engine/bookshelf/templates/view.html index e654e8ab..2f337229 100644 --- a/optional-kubernetes-engine/bookshelf/templates/view.html +++ b/optional-kubernetes-engine/bookshelf/templates/view.html @@ -36,7 +36,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/optional-kubernetes-engine/config.py b/optional-kubernetes-engine/config.py index 0fd601ed..7da9b4cc 100644 --- a/optional-kubernetes-engine/config.py +++ b/optional-kubernetes-engine/config.py @@ -78,12 +78,12 @@ # Typically, you'll name your bucket the same as your project. To create a # bucket: # -# $ gsutil mb gs:// +# $ gcloud storage buckets create gs:// # # You also need to make sure that the default ACL is set to public-read, # otherwise users will not be able to see their upload images: # -# $ gsutil defacl set public-read gs:// +# $ gcloud storage buckets update --predefined-default-object-acl=public-read gs:// # # You can adjust the max content length and allow extensions settings to allow # larger or more varied file types if desired. diff --git a/optional-kubernetes-engine/tests/conftest.py b/optional-kubernetes-engine/tests/conftest.py index d098db9d..57a50a1c 100644 --- a/optional-kubernetes-engine/tests/conftest.py +++ b/optional-kubernetes-engine/tests/conftest.py @@ -22,7 +22,7 @@ from retrying import retry -@pytest.yield_fixture(params=['datastore', 'mongodb']) +@pytest.fixture(params=['datastore', 'mongodb']) def app(request): """This fixtures provides a Flask app instance configured for testing. @@ -43,7 +43,7 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def model(monkeypatch, app): """This fixture provides a modified version of the app's model that tracks all created items and deletes them at the end of the test. diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 07729f34..00000000 --- a/renovate.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": [ - "config:base" - ], - "ignorePaths": [ - "optional-kubernetes-engine" - ] -} diff --git a/requirements.txt b/requirements.txt index e8216097..0fd66f54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -flake8==3.7.9 -nox==2019.11.9 -pytest==3.10.1 -requests==2.22.0 +flake8===5.0.4; python_version < '3.8' +flake8==6.0.0; python_version >= '3.8' +pytest==7.3.1 +nox==2023.4.22 +requests==2.31.0 diff --git a/sessions/requirements.txt b/sessions/requirements.txt index 88580978..e0e267b5 100644 --- a/sessions/requirements.txt +++ b/sessions/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-firestore==1.6.0 -flask==1.1.1 +google-cloud-firestore==2.11.1 +flask==2.2.5