diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..c0a95e72 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [ + "optional-kubernetes-engine" + ] +} diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1 @@ + diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 00000000..7d8eca8e --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,40 @@ +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: true + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: false + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: true + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "kokoro" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: + # Team slug to add to repository permissions + - team: yoshi-admins + # Access level required, one of push|pull|admin + permission: admin + - team: python-samples-reviewers + permission: admin diff --git a/.gitignore b/.gitignore index 6f2c88b4..1e32d951 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,9 @@ htmlcov/ .cache nosetests.xml coverage.xml +*_log.xml *,cover +sponge_log.xml *.log diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg index 4af0df96..f58e4f76 100644 --- a/.kokoro/common.cfg +++ b/.kokoro/common.cfg @@ -3,11 +3,26 @@ # Download trampoline resources. These will be in ${KOKORO_GFILE_DIR} gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/getting-started-python" + # All builds use the trampoline script to run in docker. -build_file: "getting-started-python/.kokoro/trampoline.sh" +build_file: "getting-started-python/.kokoro/trampoline_v2.sh" # Use the Python worker docker iamge. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:4b6ba8c199e96248980db4538065cddeea594138b9b9fb2d0388603922087747" + value: "gcr.io/cloud-devrel-kokoro-resources/python/getting-started-python" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: ".kokoro/system_tests.sh" +} + +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" } diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile new file mode 100644 index 00000000..accdd0bf --- /dev/null +++ b/.kokoro/docker/Dockerfile @@ -0,0 +1,57 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-devrel-kokoro-resources/python-base:latest + +# Install libraries needed by third-party python packages that we depend on. +RUN apt-get update \ + && apt-get install -y \ + graphviz \ + libcurl4-openssl-dev \ + libffi-dev \ + libjpeg-dev \ + libmagickwand-dev \ + libmemcached-dev \ + libmysqlclient-dev \ + libpng-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + openssl \ + zlib1g-dev \ + && apt-get clean + + +###################### Check python version + +RUN python3 --version +RUN which python3 + +# Setup Cloud SDK +ENV CLOUD_SDK_VERSION 489.0.0 +# Use system python for cloud sdk. +ENV CLOUDSDK_PYTHON python3.12 +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN /google-cloud-sdk/install.sh +ENV PATH /google-cloud-sdk/bin:$PATH + +# Setup the user profile for pip +ENV PATH ~/.local/bin:/root/.local/bin:$PATH + +# Install the current version of nox. +RUN python3 -m pip install --user --no-cache-dir nox==2022.1.7 + +CMD ["nox"] diff --git a/.kokoro/presubmit.cfg b/.kokoro/presubmit.cfg new file mode 100644 index 00000000..e69de29b diff --git a/.kokoro/system_tests.cfg b/.kokoro/system_tests.cfg index a412a3c7..e69de29b 100644 --- a/.kokoro/system_tests.cfg +++ b/.kokoro/system_tests.cfg @@ -1,10 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Download secrets from Cloud Storage. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/getting-started-python" - -# Tell the trampoline which build file to use. -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/getting-started-python/.kokoro/system_tests.sh" -} diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index 3ece8473..29bb4ae5 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -18,7 +18,8 @@ set -eo pipefail export PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin -cd github/getting-started-python +cd "${PROJECT_ROOT:-github/getting-started-python}" + # Unencrypt and extract secrets SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") @@ -27,5 +28,20 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") # Setup environment variables export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service-account.json" +# This block is executed only with Trampoline V2. +if [[ -n "${TRAMPOLINE_VERSION:-}" ]]; then + # Install nox as a user and add it to the PATH. + python3 -m pip install --user nox + export PATH="${PATH}:${HOME}/.local/bin" +fi + # Run tests +nox -s lint nox -s run_tests + +# If this is a nightly build, send the test log to the Flaky Bot. +# See https://github.com/googleapis/repo-automation-bots/tree/HEAD/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"system_tests"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot +fi diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..ef6972b4 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.10" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Flaky Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + "KOKORO_BUILD_ARTIFACTS_SUBDIR" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..17f21195 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + # We test this envvar in tests/python/test_envvar.py. + "TEST_ENV" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# The build will show some commands and docker build logs. +TRAMPOLINE_VERBOSE="true" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b3bf7316..00000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -sudo: false -language: python -env: - global: - - GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/service-account.json - - secure: S/A6XENZTy+py6kQxUX5EWgNJv3NGRaSTo4LdWVaZ7qKww1+akeMNHKvPXjp398UtsXkCWCpdy5HeLDYdmzX83uuFoPD1S/z7ilVpo/q4sRxDQmnw4diPHSpx32+XRyKpMpbjgJTMhocqZGcJFI9jxZ4QAHK3rul8ulvTojCyQf7OrePhgfBB72f/ZHho73VqmsngGyWNI6MtGGrBzMTtXlAOt8BgDhYXzPjhEWkE+OKAF9sMJqFmxUXyKkfTBfcGHDYcAOlNtvVdZGRC9NmzRvZxC5T7lQv/WdfGBAT3Zr4NSobKHK6DAv9Q7QvVgHIs7HRomgpHQ9IKnwyhR4RTYAnTtZyHe0/fUlPUXC8+Pkk634eRgYuVNMH8Cf7eRXcLRGF6wgBe9nHnx/o0w/kLbEyirQo3kVIM4Y6yfR5MhZ0Nb4oRq5tuXzqN1B5ZFyfmHhkwsJoeL5wgIGcwOxkoCTH8HqvPfW+SJuUpQC/sB89ixfTfUXYyMJxgGTOdg+1wD4IAfivGyWiUBcx06glfC3Sn+75oflgMz4M8y6zafXTzux755U5pEMa8Gw+5BqpDlkRliZwdAJ6UwHo/XHeJ2rgiTddEgSZYN0CR5XQFcEc69lauUEPkeE5fPubSgcH28xvbhWq7WJ1qdi3Wdw8RaCLOys/B6vwL8GhSJP+ROg= -before_install: -- ./decrypt-secrets.sh "$SECRETS_PASSWORD" -script: -- nox --session lint travis diff --git a/1-hello-world/app.yaml b/1-hello-world/app.yaml deleted file mode 100644 index 4e3cb73b..00000000 --- a/1-hello-world/app.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -# [START runtime] -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 -# [END runtime] diff --git a/1-hello-world/requirements.txt b/1-hello-world/requirements.txt deleted file mode 100644 index a34d076b..00000000 --- a/1-hello-world/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Flask==1.0.2 -gunicorn==19.9.0 diff --git a/1-hello-world/tox.ini b/1-hello-world/tox.ini deleted file mode 100644 index 82bf72e3..00000000 --- a/1-hello-world/tox.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -skipsdist = True -envlist = lint - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --exclude=env --import-order-style=google diff --git a/2-structured-data/app.yaml b/2-structured-data/app.yaml deleted file mode 100644 index 566e6a19..00000000 --- a/2-structured-data/app.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -#[START cloudsql_settings] -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" -#[END cloudsql_settings] diff --git a/2-structured-data/bookshelf/__init__.py b/2-structured-data/bookshelf/__init__.py deleted file mode 100644 index 419c524c..00000000 --- a/2-structured-data/bookshelf/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from flask import current_app, Flask, redirect, url_for - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model diff --git a/2-structured-data/bookshelf/crud.py b/2-structured-data/bookshelf/crud.py deleted file mode 100644 index baa3488d..00000000 --- a/2-structured-data/bookshelf/crud.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model -from flask import Blueprint, redirect, render_template, request, url_for - - -crud = Blueprint('crud', __name__) - - -# [START list] -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) -# [END list] - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -# [START add] -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) -# [END add] - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/2-structured-data/bookshelf/model_cloudsql.py b/2-structured-data/bookshelf/model_cloudsql.py deleted file mode 100644 index 39ab1bbb..00000000 --- a/2-structured-data/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -# [START model] -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/2-structured-data/bookshelf/templates/form.html b/2-structured-data/bookshelf/templates/form.html deleted file mode 100644 index af30bb17..00000000 --- a/2-structured-data/bookshelf/templates/form.html +++ /dev/null @@ -1,49 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{# [START form] #} -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - -
- -{% endblock %} -{# [END form] #} diff --git a/2-structured-data/bookshelf/templates/list.html b/2-structured-data/bookshelf/templates/list.html deleted file mode 100644 index 69beeeb3..00000000 --- a/2-structured-data/bookshelf/templates/list.html +++ /dev/null @@ -1,51 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/2-structured-data/bookshelf/templates/view.html b/2-structured-data/bookshelf/templates/view.html deleted file mode 100644 index 5f07c51a..00000000 --- a/2-structured-data/bookshelf/templates/view.html +++ /dev/null @@ -1,48 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

-
-
- -{% endblock %} diff --git a/2-structured-data/config.py b/2-structured-data/config.py deleted file mode 100644 index 660387d3..00000000 --- a/2-structured-data/config.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = \ - 'mongodb://user:password@host:27017/database' diff --git a/2-structured-data/requirements-dev.txt b/2-structured-data/requirements-dev.txt deleted file mode 100644 index 5bbc2e5d..00000000 --- a/2-structured-data/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -pytest==4.0.1 -pytest-cov==2.6.0 -retrying==1.3.3 diff --git a/2-structured-data/requirements.txt b/2-structured-data/requirements.txt deleted file mode 100644 index deaf011a..00000000 --- a/2-structured-data/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -gunicorn==19.9.0 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -oauth2client==4.1.2 -PyMongo==3.7.2 -six==1.11.0 diff --git a/2-structured-data/tests/conftest.py b/2-structured-data/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/2-structured-data/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/2-structured-data/tests/test_crud.py b/2-structured-data/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/2-structured-data/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/2-structured-data/tox.ini b/2-structured-data/tox.ini deleted file mode 100644 index 40c86660..00000000 --- a/2-structured-data/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/3-binary-data/app.yaml b/3-binary-data/app.yaml deleted file mode 100644 index d0de9391..00000000 --- a/3-binary-data/app.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/config for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/3-binary-data/bookshelf/__init__.py b/3-binary-data/bookshelf/__init__.py deleted file mode 100644 index 419c524c..00000000 --- a/3-binary-data/bookshelf/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from flask import current_app, Flask, redirect, url_for - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model diff --git a/3-binary-data/bookshelf/crud.py b/3-binary-data/bookshelf/crud.py deleted file mode 100644 index 6686798c..00000000 --- a/3-binary-data/bookshelf/crud.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - url_for - - -crud = Blueprint('crud', __name__) - - -# [START upload_image_file] -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url -# [END upload_image_file] - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - # [START image_url] - image_url = upload_image_file(request.files.get('image')) - # [END image_url] - - # [START image_url2] - if image_url: - data['imageUrl'] = image_url - # [END image_url2] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/3-binary-data/bookshelf/model_cloudsql.py b/3-binary-data/bookshelf/model_cloudsql.py deleted file mode 100644 index 96eb190a..00000000 --- a/3-binary-data/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/3-binary-data/bookshelf/templates/list.html b/3-binary-data/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/3-binary-data/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/3-binary-data/config.py b/3-binary-data/config.py deleted file mode 100644 index 7d397967..00000000 --- a/3-binary-data/config.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) diff --git a/3-binary-data/requirements-dev.txt b/3-binary-data/requirements-dev.txt deleted file mode 100644 index 5bbc2e5d..00000000 --- a/3-binary-data/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -pytest==4.0.1 -pytest-cov==2.6.0 -retrying==1.3.3 diff --git a/3-binary-data/requirements.txt b/3-binary-data/requirements.txt deleted file mode 100644 index fa332f42..00000000 --- a/3-binary-data/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 -gunicorn==19.9.0 -oauth2client==4.1.3 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -PyMongo==3.7.2 -six==1.11.0 diff --git a/3-binary-data/tests/conftest.py b/3-binary-data/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/3-binary-data/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/3-binary-data/tests/test_crud.py b/3-binary-data/tests/test_crud.py deleted file mode 100644 index b09e224b..00000000 --- a/3-binary-data/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) <= 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/3-binary-data/tests/test_storage.py b/3-binary-data/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/3-binary-data/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/3-binary-data/tox.ini b/3-binary-data/tox.ini deleted file mode 100644 index 40c86660..00000000 --- a/3-binary-data/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/4-auth/app.yaml b/4-auth/app.yaml deleted file mode 100644 index aa5ddf46..00000000 --- a/4-auth/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/4-auth/bookshelf/__init__.py b/4-auth/bookshelf/__init__.py deleted file mode 100644 index f3f4b515..00000000 --- a/4-auth/bookshelf/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -import httplib2 -# [START include] -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() -# [END include] - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # [START init_app] - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - # [END init_app] - - # [START logout] - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - # [END logout] - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -# [START request_user_info] -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google OAuth2 API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/oauth2/v3/userinfo') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: \n%s: %s", resp, content) - return None - session['profile'] = json.loads(content.decode('utf-8')) - -# [END request_user_info] diff --git a/4-auth/bookshelf/crud.py b/4-auth/bookshelf/crud.py deleted file mode 100644 index dd5c6291..00000000 --- a/4-auth/bookshelf/crud.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -# [START list_mine] -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['email'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) -# [END list_mine] - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -# [START add] -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['name'] - data['createdById'] = session['profile']['email'] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) -# [END add] - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/4-auth/bookshelf/model_cloudsql.py b/4-auth/bookshelf/model_cloudsql.py deleted file mode 100644 index c9456fcd..00000000 --- a/4-auth/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/4-auth/bookshelf/templates/form.html b/4-auth/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/4-auth/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/4-auth/bookshelf/templates/list.html b/4-auth/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/4-auth/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/4-auth/bookshelf/templates/view.html b/4-auth/bookshelf/templates/view.html deleted file mode 100644 index cfa9138e..00000000 --- a/4-auth/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.get('createdBy')|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/4-auth/config.py b/4-auth/config.py deleted file mode 100644 index 767efbd1..00000000 --- a/4-auth/config.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -# [START secret_key] -SECRET_KEY = 'secret' -# [END secret_key] - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/4-auth/requirements-dev.txt b/4-auth/requirements-dev.txt deleted file mode 100644 index 282373c0..00000000 --- a/4-auth/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -pytest==4.0.1 -pytest-cov==2.6.0 -retrying==1.3.3 -mock==2.0.0 diff --git a/4-auth/requirements.txt b/4-auth/requirements.txt deleted file mode 100644 index fa332f42..00000000 --- a/4-auth/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 -gunicorn==19.9.0 -oauth2client==4.1.3 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -PyMongo==3.7.2 -six==1.11.0 diff --git a/4-auth/tests/conftest.py b/4-auth/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/4-auth/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/4-auth/tests/test_auth.py b/4-auth/tests/test_auth.py deleted file mode 100644 index 491de9e0..00000000 --- a/4-auth/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'email': 'abc@example.com', 'name': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc@example.com' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def@example.com' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/4-auth/tests/test_crud.py b/4-auth/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/4-auth/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/4-auth/tests/test_storage.py b/4-auth/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/4-auth/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/4-auth/tox.ini b/4-auth/tox.ini deleted file mode 100644 index 40c86660..00000000 --- a/4-auth/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/5-logging/app.yaml b/5-logging/app.yaml deleted file mode 100644 index aa5ddf46..00000000 --- a/5-logging/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/5-logging/bookshelf/__init__.py b/5-logging/bookshelf/__init__.py deleted file mode 100644 index 2289afd4..00000000 --- a/5-logging/bookshelf/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # [START setup_logging] - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - # [END setup_logging] - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when debug - # is False - # [START setup_error_reporting] - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - # [END setup_error_reporting] - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google OAuth2 API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/oauth2/v3/userinfo') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: \n%s: %s", resp, content) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/5-logging/bookshelf/crud.py b/5-logging/bookshelf/crud.py deleted file mode 100644 index 834cb3d5..00000000 --- a/5-logging/bookshelf/crud.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['email'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['name'] - data['createdById'] = session['profile']['email'] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/5-logging/bookshelf/model_cloudsql.py b/5-logging/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/5-logging/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/5-logging/bookshelf/templates/form.html b/5-logging/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/5-logging/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/5-logging/bookshelf/templates/list.html b/5-logging/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/5-logging/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/5-logging/bookshelf/templates/view.html b/5-logging/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/5-logging/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/5-logging/config.py b/5-logging/config.py deleted file mode 100644 index b64eecd2..00000000 --- a/5-logging/config.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/5-logging/main.py b/5-logging/main.py deleted file mode 100644 index d5697c6c..00000000 --- a/5-logging/main.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/5-logging/requirements-dev.txt b/5-logging/requirements-dev.txt deleted file mode 100644 index 282373c0..00000000 --- a/5-logging/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -pytest==4.0.1 -pytest-cov==2.6.0 -retrying==1.3.3 -mock==2.0.0 diff --git a/5-logging/requirements.txt b/5-logging/requirements.txt deleted file mode 100644 index 32ff3033..00000000 --- a/5-logging/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 -google-cloud-logging==1.8.0 -google-cloud-error_reporting==0.30.0 -gunicorn==19.9.0 -oauth2client==4.1.3 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -PyMongo==3.7.2 -six==1.11.0 diff --git a/5-logging/tests/conftest.py b/5-logging/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/5-logging/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/5-logging/tests/test_auth.py b/5-logging/tests/test_auth.py deleted file mode 100644 index 491de9e0..00000000 --- a/5-logging/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'email': 'abc@example.com', 'name': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc@example.com' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def@example.com' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/5-logging/tests/test_crud.py b/5-logging/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/5-logging/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/5-logging/tests/test_storage.py b/5-logging/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/5-logging/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/5-logging/tox.ini b/5-logging/tox.ini deleted file mode 100644 index 40c86660..00000000 --- a/5-logging/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/6-pubsub/app.yaml b/6-pubsub/app.yaml deleted file mode 100644 index 89284aa0..00000000 --- a/6-pubsub/app.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex - -# [START entrypoint] -# Instead of using gunicorn directly, we'll use Honcho. Honcho is a python port -# of the Foreman process manager. For the default service, only the -# frontend process is needed. -entrypoint: honcho start -f /app/procfile bookshelf -# [END entrypoint] - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/6-pubsub/bookshelf/__init__.py b/6-pubsub/bookshelf/__init__.py deleted file mode 100644 index a40b4913..00000000 --- a/6-pubsub/bookshelf/__init__.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when debug - # is False - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google OAuth2 API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/oauth2/v3/userinfo') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: \n%s: %s", resp, content) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/6-pubsub/bookshelf/crud.py b/6-pubsub/bookshelf/crud.py deleted file mode 100644 index e2f4b429..00000000 --- a/6-pubsub/bookshelf/crud.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage, tasks -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['email'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['name'] - data['createdById'] = session['profile']['email'] - - book = get_model().create(data) - - # [START enqueue] - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - # [END enqueue] - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/6-pubsub/bookshelf/model_cloudsql.py b/6-pubsub/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/6-pubsub/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/6-pubsub/bookshelf/templates/form.html b/6-pubsub/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/6-pubsub/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/6-pubsub/bookshelf/templates/view.html b/6-pubsub/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/6-pubsub/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/6-pubsub/config.py b/6-pubsub/config.py deleted file mode 100644 index d40ef9bc..00000000 --- a/6-pubsub/config.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# Cloud Datastore dataset id, this is the same as your project id. -DATASTORE_DATASET_ID = PROJECT_ID - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/6-pubsub/main.py b/6-pubsub/main.py deleted file mode 100644 index 73d3a303..00000000 --- a/6-pubsub/main.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# [START books_queue] -# Make the queue available at the top-level, this allows you to run -# `psqworker main.books_queue`. We have to use the app's context because -# it contains all the configuration for plugins. -# If you were using another task queue, such as celery or rq, you can use this -# section to configure your queues to work with Flask. -with app.app_context(): - books_queue = bookshelf.tasks.get_books_queue() -# [END books_queue] - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/6-pubsub/monitor.py b/6-pubsub/monitor.py deleted file mode 100644 index 24c4fe63..00000000 --- a/6-pubsub/monitor.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START monitor] -import os -import sys - -from flask import Flask - - -# The app checks this file for the PID of the process to monitor. -PID_FILE = None - - -# Create app to handle health checks and monitor the queue worker. This will -# run alongside the worker, see procfile. -monitor_app = Flask(__name__) - - -# The health check reads the PID file created by psqworker and checks the proc -# filesystem to see if the worker is running. This same pattern can be used for -# rq and celery. -@monitor_app.route('/_ah/health') -def health(): - if not os.path.exists(PID_FILE): - return 'Worker pid not found', 503 - - with open(PID_FILE, 'r') as pidfile: - pid = pidfile.read() - - if not os.path.exists('/proc/{}'.format(pid)): - return 'Worker not running', 503 - - return 'healthy', 200 - - -@monitor_app.route('/') -def index(): - return health() - - -if __name__ == '__main__': - PID_FILE = sys.argv[1] - monitor_app.run('0.0.0.0', 8080) -# [END monitor] diff --git a/6-pubsub/procfile b/6-pubsub/procfile deleted file mode 100644 index 0c4fcec5..00000000 --- a/6-pubsub/procfile +++ /dev/null @@ -1,3 +0,0 @@ -bookshelf: gunicorn -b 0.0.0.0:$PORT main:app -worker: psqworker --pid /tmp/psq.pid main.books_queue -monitor: python monitor.py /tmp/psq.pid diff --git a/6-pubsub/requirements-dev.txt b/6-pubsub/requirements-dev.txt deleted file mode 100644 index 7e4b9ace..00000000 --- a/6-pubsub/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -mock==2.0.0 -pytest==4.0.1 -pytest-cov==2.6.0 -BeautifulSoup4==4.6.3 -requests==2.20.1 -retrying==1.3.3 diff --git a/6-pubsub/requirements.txt b/6-pubsub/requirements.txt deleted file mode 100644 index 9e6e136a..00000000 --- a/6-pubsub/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 -google-cloud-pubsub==0.39.1 -google-cloud-logging==1.8.0 -google-cloud-error_reporting==0.30.0 -gunicorn==19.9.0 -oauth2client==4.1.3 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -PyMongo==3.7.2 -six==1.11.0 -requests[security]==2.20.1 -honcho==1.0.1 -psq==0.7.0 diff --git a/6-pubsub/tests/conftest.py b/6-pubsub/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/6-pubsub/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/6-pubsub/tests/test_auth.py b/6-pubsub/tests/test_auth.py deleted file mode 100644 index 491de9e0..00000000 --- a/6-pubsub/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'email': 'abc@example.com', 'name': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc@example.com' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def@example.com' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/6-pubsub/tests/test_crud.py b/6-pubsub/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/6-pubsub/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/6-pubsub/tests/test_end_to_end.py b/6-pubsub/tests/test_end_to_end.py deleted file mode 100644 index df80e8a0..00000000 --- a/6-pubsub/tests/test_end_to_end.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -from bs4 import BeautifulSoup -import pytest -import requests -from retrying import retry - - -@pytest.mark.e2e -def test_end_to_end(): - """Tests designed to be run against live environments. - - Unlike the integration tests in the other packages, these tests are - designed to be run against fully-functional live environments. - - To run locally, start both main.py and psq_worker main.books_queue and - run this file. - - It can be run against a live environment by setting the E2E_URL - environment variables before running the tests: - - E2E_URL=http://your-app-id.appspot.com \ - nosetests tests/test_end_to_end.py - """ - - base_url = os.environ.get('E2E_URL', 'http://localhost:8080') - - book_data = { - 'title': 'a confederacy of dunces', - } - - response = requests.post(base_url + '/books/add', data=book_data) - - # There was a 302, so get the book's URL from the redirect. - book_url = response.request.url - book_id = book_url.rsplit('/', 1).pop() - - # Use retry because it will take some indeterminate time for the pub/sub - # message to be processed. - @retry(wait_exponential_multiplier=5000, stop_max_attempt_number=12) - def check_for_updated_data(): - # Check that the book's information was updated. - response = requests.get(book_url) - assert response.status_code == 200 - - soup = BeautifulSoup(response.text, 'html.parser') - - title = soup.find('h4', 'book-title').contents[0].strip() - assert re.search(r'A Confederacy of Dunces', title, re.I) - - author = soup.find('h5', 'book-author').string - assert re.search(r'John Kennedy Toole', author, re.I) - - description = soup.find('p', 'book-description').string - assert re.search(r'Ignatius', description, re.I) - - image_src = soup.find('img', 'book-image')['src'] - image = requests.get(image_src) - assert image.status_code == 200 - - try: - check_for_updated_data() - finally: - # Delete the book we created. - requests.get(base_url + '/books/{}/delete'.format(book_id)) diff --git a/6-pubsub/tests/test_storage.py b/6-pubsub/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/6-pubsub/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/6-pubsub/tox.ini b/6-pubsub/tox.ini deleted file mode 100644 index 8a04f077..00000000 --- a/6-pubsub/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report -m "not e2e" {posargs: tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST E2E_URL -setenv = PYTHONPATH={toxinidir} - - -[testenv:py27-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:py36-e2e] -basepython = python3.6 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/6-pubsub/worker.yaml b/6-pubsub/worker.yaml deleted file mode 100644 index 01ac8a4c..00000000 --- a/6-pubsub/worker.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -# [START worker] -service: worker -runtime: python -env: flex - -# Instead of using gunicorn directly, we'll use Honcho. Honcho is a python port -# of the Foreman process manager. For the worker service, both the queue worker -# and the monitor process are needed. -entrypoint: honcho start -f /app/procfile worker monitor - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" -# [END worker] - diff --git a/7-gce/bookshelf/__init__.py b/7-gce/bookshelf/__init__.py deleted file mode 100644 index 6cfda7fe..00000000 --- a/7-gce/bookshelf/__init__.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Create a health check handler. Health checks are used when running on - # Google Compute Engine by the load balancer to determine which instances - # can serve traffic. Google App Engine also uses health checking, but - # accepts any non-500 response as healthy. - @app.route('/_ah/health') - def health_check(): - return 'ok', 200 - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when Debug - # is False - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google OAuth2 API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/oauth2/v3/userinfo') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: \n%s: %s", resp, content) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/7-gce/bookshelf/crud.py b/7-gce/bookshelf/crud.py deleted file mode 100644 index b85e5d6a..00000000 --- a/7-gce/bookshelf/crud.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage, tasks -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['email'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['name'] - data['createdById'] = session['profile']['email'] - - book = get_model().create(data) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/7-gce/bookshelf/model_cloudsql.py b/7-gce/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/7-gce/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return "{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/7-gce/bookshelf/templates/list.html b/7-gce/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/7-gce/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/7-gce/bookshelf/templates/view.html b/7-gce/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/7-gce/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/7-gce/config.py b/7-gce/config.py deleted file mode 100644 index b64eecd2..00000000 --- a/7-gce/config.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/7-gce/gce/deploy.sh b/7-gce/gce/deploy.sh deleted file mode 100755 index 0d7482b4..00000000 --- a/7-gce/gce/deploy.sh +++ /dev/null @@ -1,159 +0,0 @@ -#! /bin/bash -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -ZONE=us-central1-f - -GROUP=frontend-group -TEMPLATE=$GROUP-tmpl -MACHINE_TYPE=f1-micro -IMAGE_FAMILY=debian-9 -IMAGE_PROJECT=debian-cloud -STARTUP_SCRIPT=startup-script.sh -SCOPES="userinfo-email,cloud-platform" -TAGS=http-server - -MIN_INSTANCES=1 -MAX_INSTANCES=10 -TARGET_UTILIZATION=0.6 - -SERVICE=my-app-service - -# -# Instance group setup -# - -# First we have to create an instance template. -# This template will be used by the instance group -# to create new instances. - -# [START create_template] -gcloud compute instance-templates create $TEMPLATE \ - --image-family $IMAGE_FAMILY \ - --image-project $IMAGE_PROJECT \ - --machine-type $MACHINE_TYPE \ - --scopes $SCOPES \ - --metadata-from-file startup-script=$STARTUP_SCRIPT \ - --tags $TAGS -# [END create_template] - -# Create the managed instance group. - -# [START create_group] -gcloud compute instance-groups managed \ - create $GROUP \ - --base-instance-name $GROUP \ - --size $MIN_INSTANCES \ - --template $TEMPLATE \ - --zone $ZONE -# [END create_group] - -# [START create_named_port] -gcloud compute instance-groups managed set-named-ports \ - $GROUP \ - --named-ports http:8080 \ - --zone $ZONE -# [END create_named_port] - -# -# Load Balancer Setup -# - -# A complete HTTP load balancer is structured as follows: -# -# 1) A global forwarding rule directs incoming requests to a target HTTP proxy. -# 2) The target HTTP proxy checks each request against a URL map to determine the -# appropriate backend service for the request. -# 3) The backend service directs each request to an appropriate backend based on -# serving capacity, zone, and instance health of its attached backends. The -# health of each backend instance is verified using either a health check. -# -# We'll create these resources in reverse order: -# service, health check, backend service, url map, proxy. - -# Create a health check -# The load balancer will use this check to keep track of which instances to send traffic to. -# Note that health checks will not cause the load balancer to shutdown any instances. - -# [START create_health_check] -gcloud compute http-health-checks create ah-health-check \ - --request-path /_ah/health \ - --port 8080 -# [END create_health_check] - -# Create a backend service, associate it with the health check and instance group. -# The backend service serves as a target for load balancing. - -# [START create_backend_service] -gcloud compute backend-services create $SERVICE \ - --http-health-checks ah-health-check \ - --global -# [END create_backend_service] - -# [START add_backend_service] -gcloud compute backend-services add-backend $SERVICE \ - --instance-group $GROUP \ - --instance-group-zone $ZONE \ - --global -# [END add_backend_service] - -# Create a URL map and web Proxy. The URL map will send all requests to the -# backend service defined above. - -# [START create_url_map] -gcloud compute url-maps create $SERVICE-map \ - --default-service $SERVICE -# [END create_url_map] - -# [START create_http_proxy] -gcloud compute target-http-proxies create $SERVICE-proxy \ - --url-map $SERVICE-map -# [END create_http_proxy] - -# Create a global forwarding rule to send all traffic to our proxy - -# [START create_forwarding_rule] -gcloud compute forwarding-rules create $SERVICE-http-rule \ - --global \ - --target-http-proxy $SERVICE-proxy \ - --ports=80 -# [END create_forwarding_rule] - -# -# Autoscaler configuration -# -# [START set_autoscaling] -gcloud compute instance-groups managed set-autoscaling \ - $GROUP \ - --max-num-replicas $MAX_INSTANCES \ - --target-load-balancing-utilization $TARGET_UTILIZATION \ - --zone $ZONE -# [END set_autoscaling] - -# [START create_firewall] -# Check if the firewall rule has been created in previous steps of the documentation -if gcloud compute firewall-rules list --filter="name~'default-allow-http-8080'" \ - --format="table(name)" | grep -q 'NAME'; then - echo "Firewall rule default-allow-http-8080 already exists." -else - gcloud compute firewall-rules create default-allow-http-8080 \ - --allow tcp:8080 \ - --source-ranges 0.0.0.0/0 \ - --target-tags http-server \ - --description "Allow port 8080 access to http-server" -fi - -# [END create_firewall] diff --git a/7-gce/gce/deployment_manager/bookshelf.jinja b/7-gce/gce/deployment_manager/bookshelf.jinja deleted file mode 100644 index 8a91dc0f..00000000 --- a/7-gce/gce/deployment_manager/bookshelf.jinja +++ /dev/null @@ -1,168 +0,0 @@ -{# -Copyright 2016 Google Inc. All rights reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -#} - -{# [START all] #} - -{# [START env] #} -{% set NAME = "bookshelf-" + env["deployment"] %} -{% set SERVICE = "bookshelf-" + env["deployment"] + "-frontend" %} -{# [END env] #} - -# -# Instance group setup -# - -# First we have to create an instance template. -# This template will be used by the instance group -# to create new instances. -resources: -- name : {{ NAME }} - type: compute.v1.instanceTemplate - properties: - properties: - tags: - items: - - http-server - disks: - - boot: True - type: PERSISTENT - initializeParams: - sourceImage: {{ properties['machine-image'] }} - diskSizeGb: 10 - diskType: pd-ssd - machineType: {{ properties['machine-type'] }} - serviceAccounts: - - email: default - scopes: {{ properties['scopes'] }} - metadata: - items: - - key: startup-script -{# [START startup] #} - value: | -{{imports['startup-script']|indent(14, true)}} -{# [END startup] #} - networkInterfaces: - - network: global/networks/default - accessConfigs: - - type: ONE_TO_ONE_NAT - name: External NAT - -# Creates the managed instance group. This is responsible for creating -# new instances using the instance template, as well as providing a named -# port the backend service can target -- name: {{ NAME }}-frontend-group - type: compute.v1.instanceGroupManager - properties: - instanceTemplate: $(ref.{{ NAME }}.selfLink) - baseInstanceName: frontend-group - targetSize: 3 - zone: {{ properties['zone'] }} - namedPorts: - - name: http - port: 8080 - - - -# Load Balancer Setup -# - -# A complete HTTP load balancer is structured as follows: -# -# 1) A global forwarding rule directs incoming requests to a target HTTP proxy. -# 2) The target HTTP proxy checks each request against a URL map to determine the -# appropriate backend service for the request. -# 3) The backend service directs each request to an appropriate backend based on -# serving capacity, zone, and instance health of its attached backends. The -# health of each backend instance is verified using either a health check. -# -# We'll create these resources in reverse order: -# service, health check, backend service, url map, proxy. - -# Create a health check -# The load balancer will use this check to keep track of which instances to send traffic to. -# Note that health checks will not cause the load balancer to shutdown any instances. -- name: {{ NAME }}-health-check - type: compute.v1.httpHealthCheck - properties: - requestPath: /_ah/health - port: 8080 - -# Create a backend service, associate it with the health check and instance group. -# The backend service serves as a target for load balancing. -- name: {{ SERVICE }} - type: compute.v1.backendService - properties: - healthChecks: - - $(ref.{{ NAME }}-health-check.selfLink) - portName: http - backends: -{# [START reference] #} - - group: $(ref.{{ NAME }}-frontend-group.instanceGroup) - zone: {{ properties['zone'] }} -{# [END reference] #} - -# Create a URL map and web Proxy. The URL map will send all requests to the -# backend service defined above. -- name: {{ SERVICE }}-map - type: compute.v1.urlMap - properties: - defaultService: $(ref.{{ SERVICE }}.selfLink) - -# This is the actual proxy which uses the URL map to route traffic -# to the backend service -- name: {{ SERVICE }}-proxy - type: compute.v1.targetHttpProxy - properties: - urlMap: $(ref.{{ SERVICE }}-map.selfLink) - -# This is the global forwarding rule which creates an external IP to -# target the http poxy -- name: {{ SERVICE }}-http-rule - type: compute.v1.globalForwardingRule - properties: - target: $(ref.{{ SERVICE }}-proxy.selfLink) - portRange: 80 - -# Creates an autoscaler resource (note that when using the gcloud CLI, -# autoscaling is set as a configuration of the managed instance group -# but autoscaler is a resource so in deployment manager we explicitly -# define it -- name: {{ NAME }}-autoscaler - type: compute.v1.autoscaler - properties: - zone: {{ properties['zone'] }} - target: $(ref.{{ NAME }}-frontend-group.selfLink) - autoscalingPolicy: -{# [START properties] #} - minNumReplicas: {{ properties['min-instances'] }} - maxNumReplicas: {{ properties['max-instances'] }} - loadBalancingUtilization: - utilizationTarget: {{ properties['target-utilization'] }} -{# [END properties] #} - -# Firewall rule that allows traffic to GCE instances with the -# http server tag we created -- name: {{ NAME }}-allow-http - type: compute.v1.firewall - properties: - allowed: - - IPProtocol: tcp - ports: - - 8080 - sourceRanges: - - 0.0.0.0/0 - targetTags: - - http-server - description: "Allow port 8080 access to http-server" - -{# [END all] #} diff --git a/7-gce/gce/deployment_manager/bookshelf.jinja.schema b/7-gce/gce/deployment_manager/bookshelf.jinja.schema deleted file mode 100644 index 526deef6..00000000 --- a/7-gce/gce/deployment_manager/bookshelf.jinja.schema +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START all] - -info: - title: Bookshelf GCE Deploy - author: Google Inc. - description: Creates a GCE Deployment - -imports: -- name: startup-script - path: ../startup-script.sh - -required: -- zone -- machine-type -- min-instances -- max-instances -- scopes - -properties: - zone: - description: Zone to create the resources in. - type: string - machine-type: - description: Type of machine to use - type: string - machine-image: - description: The OS image to use on the machines - type: string - min-instances: - description: The minimum number of VMs the autoscaler will create - type: integer - max-instances: - description: The maximum number of VMs the autoscaler will create - type: integer - target-utilization: - description: The target CPU usage for the autoscaler to base its scaling on - type: number - scopes: - description: A list of scopes to create the VM with - type: array - minItems: 1 - items: - type: string - -# [END all] diff --git a/7-gce/gce/deployment_manager/config.yaml b/7-gce/gce/deployment_manager/config.yaml deleted file mode 100644 index 901b55fb..00000000 --- a/7-gce/gce/deployment_manager/config.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START all] -imports: -- name: bookshelf.jinja - path: ./bookshelf.jinja - -resources: -- name: bookshelf - type: bookshelf.jinja - properties: - zone: us-central1-f - machine-type: n1-standard-1 - machine-image: https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/family/debian-9 - min-instances: 1 - max-instances: 10 - target-utilization: 0.6 - scopes: - - https://www.googleapis.com/auth/cloud-platform - -# [END all] diff --git a/7-gce/gce/startup-script.sh b/7-gce/gce/startup-script.sh deleted file mode 100644 index 6f05e7ce..00000000 --- a/7-gce/gce/startup-script.sh +++ /dev/null @@ -1,76 +0,0 @@ -#! /bin/bash -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START startup] -set -v - -# Talk to the metadata server to get the project id -PROJECTID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") - -# Install logging monitor. The monitor will automatically pickup logs sent to -# syslog. -# [START logging] -curl -s "https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh" | bash -service google-fluentd restart & -# [END logging] - -# Install dependencies from apt -apt-get update -apt-get install -yq \ - git build-essential supervisor python python-dev python-pip libffi-dev \ - libssl-dev - -# Create a pythonapp user. The application will run as this user. -useradd -m -d /home/pythonapp pythonapp - -# pip from apt is out of date, so make it update itself and install virtualenv. -pip install --upgrade pip virtualenv - -# Get the source code from the Google Cloud Repository -# git requires $HOME and it's not set during the startup script. -export HOME=/root -git config --global credential.helper gcloud.sh -git clone https://source.developers.google.com/p/$PROJECTID/r/[YOUR_REPO_NAME] /opt/app - -# Install app dependencies -virtualenv -p python3 /opt/app/7-gce/env -source /opt/app/7-gce/env/bin/activate -/opt/app/7-gce/env/bin/pip install -r /opt/app/7-gce/requirements.txt - -# Make sure the pythonapp user owns the application code -chown -R pythonapp:pythonapp /opt/app - -# Configure supervisor to start gunicorn inside of our virtualenv and run the -# application. -cat >/etc/supervisor/conf.d/python-app.conf << EOF -[program:pythonapp] -directory=/opt/app/7-gce -command=/opt/app/7-gce/env/bin/honcho start -f ./procfile worker bookshelf -autostart=true -autorestart=true -user=pythonapp -# Environment variables ensure that the application runs inside of the -# configured virtualenv. -environment=VIRTUAL_ENV="/opt/app/7-gce/env",PATH="/opt/app/7-gce/env/bin",\ - HOME="/home/pythonapp",USER="pythonapp" -stdout_logfile=syslog -stderr_logfile=syslog -EOF - -supervisorctl reread -supervisorctl update - -# Application should now be running under supervisor -# [END startup] diff --git a/7-gce/gce/teardown.sh b/7-gce/gce/teardown.sh deleted file mode 100755 index 8f123a10..00000000 --- a/7-gce/gce/teardown.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#! /bin/bash - -set -x - -ZONE=us-central1-f -gcloud config set compute/zone $ZONE - -GROUP=frontend-group -TEMPLATE=$GROUP-tmpl -SERVICE=my-app-service - -gcloud compute instance-groups managed stop-autoscaling $GROUP --zone $ZONE - -gcloud compute forwarding-rules delete $SERVICE-http-rule --global --quiet - -gcloud compute target-http-proxies delete $SERVICE-proxy --quiet - -gcloud compute url-maps delete $SERVICE-map --quiet - -gcloud compute backend-services delete $SERVICE --global --quiet - -gcloud compute http-health-checks delete ah-health-check --quiet - -gcloud compute instance-groups managed delete $GROUP --quiet - -gcloud compute instance-templates delete $TEMPLATE --quiet - -gcloud compute firewall-rules delete default-allow-http-8080 --quiet diff --git a/7-gce/main.py b/7-gce/main.py deleted file mode 100644 index 3975fab6..00000000 --- a/7-gce/main.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# Make the queue available at the top-level, this allows you to run -# `psqworker main.books_queue`. We have to use the app's context because -# it contains all the configuration for plugins. -# If you were using another task queue, such as celery or rq, you can use this -# section to configure your queues to work with Flask. -with app.app_context(): - books_queue = bookshelf.tasks.get_books_queue() - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/7-gce/monitor.py b/7-gce/monitor.py deleted file mode 100644 index ecbd098b..00000000 --- a/7-gce/monitor.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from flask import Flask - - -# The app checks this file for the PID of the process to monitor. -PID_FILE = None - - -# Create app to handle health checks and monitor the queue worker. This will -# run alongside the worker, see procfile. -monitor_app = Flask(__name__) - - -# The health check reads the PID file created by psqworker and checks the proc -# filesystem to see if the worker is running. This same pattern can be used for -# rq and celery. -@monitor_app.route('/_ah/health') -def health(): - if not os.path.exists(PID_FILE): - return 'Worker pid not found', 503 - - with open(PID_FILE, 'r') as pidfile: - pid = pidfile.read() - - if not os.path.exists('/proc/{}'.format(pid)): - return 'Worker not running', 503 - - return 'healthy', 200 - - -@monitor_app.route('/') -def index(): - return health() - - -if __name__ == '__main__': - PID_FILE = sys.argv[1] - monitor_app.run('0.0.0.0', 8080) diff --git a/7-gce/procfile b/7-gce/procfile deleted file mode 100644 index 0c4fcec5..00000000 --- a/7-gce/procfile +++ /dev/null @@ -1,3 +0,0 @@ -bookshelf: gunicorn -b 0.0.0.0:$PORT main:app -worker: psqworker --pid /tmp/psq.pid main.books_queue -monitor: python monitor.py /tmp/psq.pid diff --git a/7-gce/requirements-dev.txt b/7-gce/requirements-dev.txt deleted file mode 100644 index 7e4b9ace..00000000 --- a/7-gce/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -tox==3.5.3 -flake8==3.6.0 -flaky==3.4.0 -mock==2.0.0 -pytest==4.0.1 -pytest-cov==2.6.0 -BeautifulSoup4==4.6.3 -requests==2.20.1 -retrying==1.3.3 diff --git a/7-gce/requirements.txt b/7-gce/requirements.txt deleted file mode 100644 index 94c7387b..00000000 --- a/7-gce/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -Flask>=1.0.0 -google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 -google-cloud-logging==1.8.0 -google-cloud-error_reporting==0.30.0 -gunicorn==19.9.0 -oauth2client==4.1.3 -Flask-SQLAlchemy==2.3.2 -PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 -PyMongo==3.7.2 -six==1.11.0 -requests[security]==2.20.1 -honcho==1.0.1 -psq==0.7.0 diff --git a/7-gce/tests/conftest.py b/7-gce/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/7-gce/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/7-gce/tests/test_auth.py b/7-gce/tests/test_auth.py deleted file mode 100644 index 72da036e..00000000 --- a/7-gce/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'email': 'abc@example.com', 'name': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/mine') - - assert rv.status < '400' - body = rv.data.decode('utf-8') - assert 'Redirecting' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status < '400' - body = rv.data.decode('utf-8') - assert 'Redirecting' not in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc@example.com' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def@example.com' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/7-gce/tests/test_crud.py b/7-gce/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/7-gce/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/7-gce/tests/test_end_to_end.py b/7-gce/tests/test_end_to_end.py deleted file mode 100644 index df80e8a0..00000000 --- a/7-gce/tests/test_end_to_end.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -from bs4 import BeautifulSoup -import pytest -import requests -from retrying import retry - - -@pytest.mark.e2e -def test_end_to_end(): - """Tests designed to be run against live environments. - - Unlike the integration tests in the other packages, these tests are - designed to be run against fully-functional live environments. - - To run locally, start both main.py and psq_worker main.books_queue and - run this file. - - It can be run against a live environment by setting the E2E_URL - environment variables before running the tests: - - E2E_URL=http://your-app-id.appspot.com \ - nosetests tests/test_end_to_end.py - """ - - base_url = os.environ.get('E2E_URL', 'http://localhost:8080') - - book_data = { - 'title': 'a confederacy of dunces', - } - - response = requests.post(base_url + '/books/add', data=book_data) - - # There was a 302, so get the book's URL from the redirect. - book_url = response.request.url - book_id = book_url.rsplit('/', 1).pop() - - # Use retry because it will take some indeterminate time for the pub/sub - # message to be processed. - @retry(wait_exponential_multiplier=5000, stop_max_attempt_number=12) - def check_for_updated_data(): - # Check that the book's information was updated. - response = requests.get(book_url) - assert response.status_code == 200 - - soup = BeautifulSoup(response.text, 'html.parser') - - title = soup.find('h4', 'book-title').contents[0].strip() - assert re.search(r'A Confederacy of Dunces', title, re.I) - - author = soup.find('h5', 'book-author').string - assert re.search(r'John Kennedy Toole', author, re.I) - - description = soup.find('p', 'book-description').string - assert re.search(r'Ignatius', description, re.I) - - image_src = soup.find('img', 'book-image')['src'] - image = requests.get(image_src) - assert image.status_code == 200 - - try: - check_for_updated_data() - finally: - # Delete the book we created. - requests.get(base_url + '/books/{}/delete'.format(book_id)) diff --git a/7-gce/tests/test_storage.py b/7-gce/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/7-gce/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/7-gce/tox.ini b/7-gce/tox.ini deleted file mode 100644 index 8a04f077..00000000 --- a/7-gce/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py36 - -[testenv] -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report -m "not e2e" {posargs: tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST E2E_URL -setenv = PYTHONPATH={toxinidir} - - -[testenv:py27-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:py36-e2e] -basepython = python3.6 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0dfefd93 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The python-samples-owners team is the default owner for anything not +# explicitly taken by someone else. +* @GoogleCloudPlatform/python-samples-reviewers diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6736efd9..c42ef953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,14 +6,12 @@ We'd love to accept your sample apps and patches! Before we can take them, we have to jump a couple of legal hurdles. Please fill out either the individual or corporate Contributor License Agreement -(CLA). +(CLA): * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual CLA] - (https://developers.google.com/open-source/cla/individual). + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA] - (https://developers.google.com/open-source/cla/corporate). + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to @@ -21,15 +19,14 @@ accept your pull requests. ## Contributing A Patch -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owner will respond to your issue promptly. +1. Submit an issue describing your proposed change to the repository in question. +1. The repository owner will respond to your issue promptly. 1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the desired repo, develop and test your code changes. + CLA (see details above). +1. Fork the desired repo, then develop and test your code changes. 1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + you are contributing. Refer to the [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md) and the + [Google Cloud Platform Community Style Guide](https://cloud.google.com/community/tutorials/styleguide) for the recommended coding standards for this organization. 1. Ensure that your code has an appropriate set of unit tests which all pass. 1. Submit a pull request. diff --git a/README.md b/README.md index 85d0c8f8..a347e9bb 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,22 @@ # Getting started with Python on Google Cloud Platform -[![Build Status](https://travis-ci.org/GoogleCloudPlatform/getting-started-python.svg)](https://travis-ci.org/GoogleCloudPlatform/getting-started-python) - -This repository is the complete sample code for the [Python Getting Started on Google Cloud Platform](http://cloud.google.com/python) tutorials. Please refer to the tutorials for instructions on configuring, running, and deploying these samples. +This repository is the complete sample code for the [Python Getting Started on Google Cloud Platform](https://cloud.google.com/python/docs/) tutorials. Please refer to the tutorials for instructions on configuring, running, and deploying these samples. The code for the samples is contained in individual folders in this repository. -Note that 7-gce is the final version of the project, including the complete Bookshelf app on Managed VMs in addition to scripts to alternatively deploy on GCE. - - Tutorial | Folder ---------|------- -[Hello world](https://cloud.google.com/python/getting-started/hello-world) | [1-hello-world](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/1-hello-world) -[Structured data](https://cloud.google.com/python/getting-started/using-structured-data) | [2-structured-data](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/2-structured-data) -[Cloud Storage](https://cloud.google.com/python/getting-started/using-cloud-storage) | [3-binary-data](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/3-binary-data) -[Authenticating users](https://cloud.google.com/python/getting-started/authenticate-users) | [4-auth](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/4-auth) -[Logging app events](https://cloud.google.com/python/monitor-and-debug/logging-application-events) | [5-logging](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/5-logging) -[Using Cloud Pub/Sub](https://cloud.google.com/python/getting-started/using-pub-sub) | [6-pubsub](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/6-pubsub) -[Deploying to Google Compute Engine](https://cloud.google.com/python/getting-started/run-on-compute-engine) | [7-gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/7-gce) +[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/bookshelf) +[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/background) +[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/quickstart/python) +[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/gce) +[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/sessions) +[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/authenticating-users) ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) - ## Licensing * See [LICENSE](LICENSE) diff --git a/2-structured-data/main.py b/authenticating-users/app.yaml similarity index 66% rename from 2-structured-data/main.py rename to authenticating-users/app.yaml index d5697c6c..9010f5b3 100644 --- a/2-structured-data/main.py +++ b/authenticating-users/app.yaml @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) +# [START getting_started_app_yaml] +runtime: python37 +# [END getting_started_app_yaml] diff --git a/authenticating-users/main.py b/authenticating-users/main.py new file mode 100644 index 00000000..9f4b9504 --- /dev/null +++ b/authenticating-users/main.py @@ -0,0 +1,112 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_auth_all] +import sys + +from flask import Flask +app = Flask(__name__) + +CERTS = None +AUDIENCE = None + + +# [START getting_started_auth_certs] +def certs(): + """Returns a dictionary of current Google public key certificates for + validating Google-signed JWTs. Since these change rarely, the result + is cached on first request for faster subsequent responses. + """ + import requests + + global CERTS + if CERTS is None: + response = requests.get( + 'https://www.gstatic.com/iap/verify/public_key' + ) + CERTS = response.json() + return CERTS +# [END getting_started_auth_certs] + + +# [START getting_started_auth_metadata] +def get_metadata(item_name): + """Returns a string with the project metadata value for the item_name. + See https://cloud.google.com/compute/docs/storing-retrieving-metadata for + possible item_name values. + """ + import requests + + endpoint = 'http://metadata.google.internal' + path = '/computeMetadata/v1/project/' + path += item_name + response = requests.get( + '{}{}'.format(endpoint, path), + headers={'Metadata-Flavor': 'Google'} + ) + metadata = response.text + return metadata +# [END getting_started_auth_metadata] + + +# [START getting_started_auth_audience] +def audience(): + """Returns the audience value (the JWT 'aud' property) for the current + running instance. Since this involves a metadata lookup, the result is + cached when first requested for faster future responses. + """ + global AUDIENCE + if AUDIENCE is None: + project_number = get_metadata('numeric-project-id') + project_id = get_metadata('project-id') + AUDIENCE = '/projects/{}/apps/{}'.format( + project_number, project_id + ) + return AUDIENCE +# [END getting_started_auth_audience] + + +# [START getting_started_auth_validate_assertion] +def validate_assertion(assertion): + """Checks that the JWT assertion is valid (properly signed, for the + correct audience) and if so, returns strings for the requesting user's + email and a persistent user ID. If not valid, returns None for each field. + """ + from jose import jwt + + try: + info = jwt.decode( + assertion, + certs(), + algorithms=['ES256'], + audience=audience() + ) + return info['email'], info['sub'] + except Exception as e: + print('Failed to validate assertion: {}'.format(e), file=sys.stderr) + return None, None +# [END getting_started_auth_validate_assertion] + + +# [START getting_started_auth_front_controller] +@app.route('/', methods=['GET']) +def say_hello(): + from flask import request + + assertion = request.headers.get('X-Goog-IAP-JWT-Assertion') + email, id = validate_assertion(assertion) + page = "

Hello {}

".format(email) + return page +# [END getting_started_auth_front_controller] +# [END getting_started_auth_all] diff --git a/authenticating-users/main_test.py b/authenticating-users/main_test.py new file mode 100644 index 00000000..38b934b6 --- /dev/null +++ b/authenticating-users/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def fake_validate(assertion): + if assertion == "Valid": + return "nobody@example.com", "user0001" + else: + return None, None + + +main.validate_assertion = fake_validate + + +def test_home_page(): + client = main.app.test_client() + + # Good request check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Valid"}) + assert "nobody@example.com" in r.text + + # Missing header check + r = client.get("/") + assert "None" in r.text + + # Bad header check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Not Valid"}) + assert "None" in r.text diff --git a/authenticating-users/requirements-test.txt b/authenticating-users/requirements-test.txt new file mode 100644 index 00000000..6a3d7bca --- /dev/null +++ b/authenticating-users/requirements-test.txt @@ -0,0 +1 @@ +pytest==7.1.2 \ No newline at end of file diff --git a/authenticating-users/requirements.txt b/authenticating-users/requirements.txt new file mode 100644 index 00000000..0d4a3c36 --- /dev/null +++ b/authenticating-users/requirements.txt @@ -0,0 +1,6 @@ +# [START getting_started_requirements] +Flask==2.2.5 +cryptography==41.0.2 +python-jose[cryptography]==3.3.0 +requests==2.31.0 +# [END getting_started_requirements] diff --git a/background/README.md b/background/README.md new file mode 100644 index 00000000..d79c95fe --- /dev/null +++ b/background/README.md @@ -0,0 +1,17 @@ +Background Processing +--------------------- + +This directory contains an example of doing background processing with App +Engine, Cloud Pub/Sub, Cloud Functions, and Firestore. + +Deploy commands: + +From the app directory: +``` +$ gcloud app deploy +``` + +From the function directory, after creating the PubSub topic: +``` +$ gcloud functions deploy --runtime=python37 --trigger-topic=translate Translate --set-env-vars GOOGLE_CLOUD_PROJECT=my-project +``` diff --git a/background/app/app.yaml b/background/app/app.yaml new file mode 100644 index 00000000..02c0651c --- /dev/null +++ b/background/app/app.yaml @@ -0,0 +1,17 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_background_config] +runtime: python312 +# [END getting_started_background_config] diff --git a/background/app/main.py b/background/app/main.py new file mode 100644 index 00000000..fad93d46 --- /dev/null +++ b/background/app/main.py @@ -0,0 +1,88 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This web app shows translations that have been previously requested, and + provides a form to request a new translation. +""" + +# [START getting_started_background_app_main] +import json +import os + +from flask import Flask, redirect, render_template, request +from google.cloud import firestore, pubsub +from markupsafe import escape + + +app = Flask(__name__) + +# Get client objects to reuse over multiple invocations +db = firestore.Client() +publisher = pubsub.PublisherClient() + +# Keep this list of supported languages up to date +ACCEPTABLE_LANGUAGES = ("de", "en", "es", "fr", "ja", "sw") +# [END getting_started_background_app_main] + + +# [START getting_started_background_app_list] +@app.route("/", methods=["GET"]) +def index(): + """The home page has a list of prior translations and a form to + ask for a new translation. + """ + + doc_list = [] + docs = db.collection("translations").stream() + for doc in docs: + doc_list.append(doc.to_dict()) + + return render_template("index.html", translations=doc_list) + + +# [END getting_started_background_app_list] + + +# [START getting_started_background_app_request] +@app.route("/request-translation", methods=["POST"]) +def translate(): + """Handle a request to translate a string (form field 'v') to a given + language (form field 'lang'), by sending a PubSub message to a topic. + """ + source_string = request.form.get("v", "") + to_language = escape(request.form.get("lang", "")) + + if source_string == "": + return "Invalid request, you must provide a value.", 400 + + if to_language not in ACCEPTABLE_LANGUAGES: + return f"Unsupported language: {to_language}", 400 + + message = { + "Original": source_string, + "Language": to_language, + "Translated": "", + "OriginalLanguage": "", + } + + topic_name = ( + f"projects/{os.getenv('GOOGLE_CLOUD_PROJECT')}/topics/translate" + ) + publisher.publish( + topic=topic_name, data=json.dumps(message).encode("utf-8") + ) + return redirect("/") + + +# [END getting_started_background_app_request] diff --git a/background/app/main_test.py b/background/app/main_test.py new file mode 100644 index 00000000..41b529d3 --- /dev/null +++ b/background/app/main_test.py @@ -0,0 +1,111 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import google.auth +from google.cloud import firestore, pubsub, storage +import main +import pytest + + +credentials, project_id = google.auth.default() +os.environ["GOOGLE_CLOUD_PROJECT"] = project_id +SUBSCRIPTION_NAME = "projects/{}/subscriptions/{}".format( + project_id, "test-" + str(uuid.uuid4()) +) +TOPIC_NAME = "projects/{}/topics/{}".format(project_id, "translate") + + +@pytest.fixture +def db(): + def clear_collection(collection): + """Removes every document from the collection, to make it easy to see + what has been added by the current test run. + """ + for doc in collection.stream(): + doc.reference.delete() + + bucket_name = 'system-test-bucket' + client = firestore.Client() + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + translations = client.collection("translations") + clear_collection(translations) + translations.add( + { + "Original": "A testing message", + "Language": "fr", + "Translated": '"A testing message", but in French', + "OriginalLanguage": "en", + }, + document_id="test translation", + ) + assert bucket in locals() + yield client + + +@pytest.fixture +def publisher(): + client = pubsub.PublisherClient() + yield client + + +@pytest.fixture +def subscriber(): + subscriber = pubsub.SubscriberClient() + subscriber.create_subscription( + request={"name": SUBSCRIPTION_NAME, "topic": TOPIC_NAME} + ) + yield subscriber + subscriber.delete_subscription(request={"subscription": SUBSCRIPTION_NAME}) + + +def test_index(db, publisher): + main.app.testing = True + main.db = db + main.publisher = publisher + client = main.app.test_client() + + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Text to translate" in response_text + assert "but in French" in response_text + + +def test_translate(db, publisher, subscriber): + main.app.testing = True + main.db = db + main.publisher = publisher + client = main.app.test_client() + + r = client.post( + "/request-translation", + data={ + "v": "This is a test", + "lang": "fr", + }, + ) + + assert r.status_code < 400 + + response = subscriber.pull( + request={"subscription": SUBSCRIPTION_NAME, "max_messages": 1}, + timeout=10.0, + ) + assert len(response.received_messages) == 1 + assert b"This is a test" in response.received_messages[0].message.data + assert b"fr" in response.received_messages[0].message.data diff --git a/background/app/requirements.txt b/background/app/requirements.txt new file mode 100644 index 00000000..f70d16b7 --- /dev/null +++ b/background/app/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-firestore==2.18.0 +google-cloud-pubsub==2.23.0 +flask==3.0.3 diff --git a/background/app/templates/index.html b/background/app/templates/index.html new file mode 100644 index 00000000..6f30e24d --- /dev/null +++ b/background/app/templates/index.html @@ -0,0 +1,145 @@ + + + + + + + + + + Translations + + + + + + + + + + + +
+
+
+ + Translate with Background Processing +
+
+
+
+
+
+
+
+
+ + +
+ + +
+
+
+ + + + + + + + + {% for translation in translations %} + + + + + {% endfor %} + +
OriginalTranslation
+ + {{ translation['OriginalLanguage'] }} + + {{ translation['Original'] }} + + + {{ translation['Language'] }} + + {{ translation['Translated'] }} +
+
+ +
+
+
+
+
+ +
+
+
+ + + + diff --git a/background/function/main.py b/background/function/main.py new file mode 100644 index 00000000..ccf94244 --- /dev/null +++ b/background/function/main.py @@ -0,0 +1,106 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This function handles messages posted to a pubsub topic by translating + the data in the message as requested. The message must be a JSON encoded + dictionary with fields: + + Original - the string to translate + Language - the language to translate the string to + + The dictionary may have other fields, which will be ignored. +""" + +# [START getting_started_background_translate_setup] +import base64 +import hashlib +import json + +from google.cloud import firestore +from google.cloud import translate_v2 as translate +# [END getting_started_background_translate_setup] + +# [START getting_started_background_translate_init] +# Get client objects once to reuse over multiple invocations. +xlate = translate.Client() +db = firestore.Client() +# [END getting_started_background_translate_init] + + +# [START getting_started_background_translate_string] +def translate_string(from_string, to_language): + """ Translates a string to a specified language. + + from_string - the original string before translation + + to_language - the language to translate to, as a two-letter code (e.g., + 'en' for english, 'de' for german) + + Returns the translated string and the code for original language + """ + result = xlate.translate(from_string, target_language=to_language) + return result['translatedText'], result['detectedSourceLanguage'] +# [END getting_started_background_translate_string] + + +# [START getting_started_background_translate] +def document_name(message): + """ Messages are saved in a Firestore database with document IDs generated + from the original string and destination language. If the exact same + translation is requested a second time, the result will overwrite the + prior result. + + message - a dictionary with fields named Language and Original, and + optionally other fields with any names + + Returns a unique name that is an allowed Firestore document ID + """ + key = '{}/{}'.format(message['Language'], message['Original']) + hashed = hashlib.sha512(key.encode()).digest() + + # Note that document IDs should not contain the '/' character + name = base64.b64encode(hashed, altchars=b'+-').decode('utf-8') + return name + + +@firestore.transactional +def update_database(transaction, message): + name = document_name(message) + doc_ref = db.collection('translations').document(document_id=name) + + try: + doc_ref.get(transaction=transaction) + except firestore.NotFound: + return # Don't replace an existing translation + + transaction.set(doc_ref, message) + + +def translate_message(event, context): + """ Process a pubsub message requesting a translation + """ + message_data = base64.b64decode(event['data']).decode('utf-8') + message = json.loads(message_data) + + from_string = message['Original'] + to_language = message['Language'] + + to_string, from_language = translate_string(from_string, to_language) + + message['Translated'] = to_string + message['OriginalLanguage'] = from_language + + transaction = db.transaction() + update_database(transaction, message) +# [END getting_started_background_translate] diff --git a/background/function/main_test.py b/background/function/main_test.py new file mode 100644 index 00000000..aa6d3ddb --- /dev/null +++ b/background/function/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +from google.cloud import firestore +import main + + +def clear_collection(collection): + """ Removes every document from the collection, to make it easy to see + what has been added by the current test run. + """ + for doc in collection.stream(): + doc.reference.delete() + + +def test_invocations(): + db = firestore.Client() + main.db = db + + translations = db.collection('translations') + clear_collection(translations) + + event = { + 'data': base64.b64encode(json.dumps({ + 'Original': 'My test message', + 'Language': 'de', + }).encode('utf-8')) + } + + main.translate_message(event, None) + + docs = [doc for doc in translations.stream()] + assert len(docs) == 1 # Should be only the one just created + + message = docs[0].to_dict() + + assert message['Original'] == 'My test message' + assert message['Language'] == 'de' + assert len(message['Translated']) > 0 + assert message['OriginalLanguage'] == 'en' diff --git a/background/function/requirements.txt b/background/function/requirements.txt new file mode 100644 index 00000000..b8e6aaad --- /dev/null +++ b/background/function/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-translate==3.11.1 +google-cloud-firestore==2.11.1 diff --git a/bookshelf/Dockerfile b/bookshelf/Dockerfile new file mode 100644 index 00000000..44c98b38 --- /dev/null +++ b/bookshelf/Dockerfile @@ -0,0 +1,14 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.11-slim + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install --no-cache-dir -r requirements.txt + +# Run the web service on container startup. +ENTRYPOINT [ "gunicorn", "--bind", "0.0.0.0:8080", "main:app" ] \ No newline at end of file diff --git a/bookshelf/app.yaml b/bookshelf/app.yaml new file mode 100644 index 00000000..ac4c378f --- /dev/null +++ b/bookshelf/app.yaml @@ -0,0 +1,2 @@ +runtime: python37 + diff --git a/bookshelf/firestore.py b/bookshelf/firestore.py new file mode 100644 index 00000000..43f18b25 --- /dev/null +++ b/bookshelf/firestore.py @@ -0,0 +1,69 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bookshelf_firestore_client_import] +from google.cloud import firestore +# [END bookshelf_firestore_client_import] + + +def document_to_dict(doc): + if not doc.exists: + return None + doc_dict = doc.to_dict() + doc_dict['id'] = doc.id + return doc_dict + + +def next_page(limit=10, start_after=None): + db = firestore.Client() + + query = db.collection(u'Book').limit(limit).order_by(u'title') + + if start_after: + # Construct a new query starting at this document. + query = query.start_after({u'title': start_after}) + + docs = query.stream() + docs = list(map(document_to_dict, docs)) + + last_title = None + if limit == len(docs): + # Get the last document from the results and set as the last title. + last_title = docs[-1][u'title'] + return docs, last_title + + +def read(book_id): + # [START bookshelf_firestore_client] + db = firestore.Client() + book_ref = db.collection(u'Book').document(book_id) + snapshot = book_ref.get() + # [END bookshelf_firestore_client] + return document_to_dict(snapshot) + + +def update(data, book_id=None): + db = firestore.Client() + book_ref = db.collection(u'Book').document(book_id) + book_ref.set(data) + return document_to_dict(book_ref.get()) + + +create = update + + +def delete(id): + db = firestore.Client() + book_ref = db.collection(u'Book').document(id) + book_ref.delete() diff --git a/bookshelf/images/moby-dick.png b/bookshelf/images/moby-dick.png new file mode 100755 index 00000000..ef789cdb Binary files /dev/null and b/bookshelf/images/moby-dick.png differ diff --git a/bookshelf/main.py b/bookshelf/main.py new file mode 100644 index 00000000..789bec2e --- /dev/null +++ b/bookshelf/main.py @@ -0,0 +1,153 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import firestore +from flask import current_app, flash, Flask, Markup, redirect, render_template +from flask import request, url_for +from google.cloud import error_reporting +import google.cloud.logging +import storage + + +# [START upload_image_file] +def upload_image_file(img): + """ + Upload the user-uploaded file to Google Cloud Storage and retrieve its + publicly-accessible URL. + """ + if not img: + return None + + public_url = storage.upload_file( + img.read(), + img.filename, + img.content_type + ) + + current_app.logger.info( + 'Uploaded file %s as %s.', img.filename, public_url) + + return public_url +# [END upload_image_file] + + +app = Flask(__name__) +app.config.update( + SECRET_KEY='secret', + MAX_CONTENT_LENGTH=8 * 1024 * 1024, + ALLOWED_EXTENSIONS=set(['png', 'jpg', 'jpeg', 'gif']) +) + +app.debug = False +app.testing = False + +# Configure logging +if not app.testing: + logging.basicConfig(level=logging.INFO) + client = google.cloud.logging.Client() + # Attaches a Google Stackdriver logging handler to the root logger + client.setup_logging() + + +@app.route('/') +def list(): + start_after = request.args.get('start_after', None) + books, last_title = firestore.next_page(start_after=start_after) + + return render_template('list.html', books=books, last_title=last_title) + + +@app.route('/books/') +def view(book_id): + book = firestore.read(book_id) + return render_template('view.html', book=book) + + +@app.route('/books/add', methods=['GET', 'POST']) +def add(): + if request.method == 'POST': + data = request.form.to_dict(flat=True) + + # If an image was uploaded, update the data to point to the new image. + image_url = upload_image_file(request.files.get('image')) + + if image_url: + data['imageUrl'] = image_url + + book = firestore.create(data) + + return redirect(url_for('.view', book_id=book['id'])) + + return render_template('form.html', action='Add', book={}) + + +@app.route('/books//edit', methods=['GET', 'POST']) +def edit(book_id): + book = firestore.read(book_id) + + if request.method == 'POST': + data = request.form.to_dict(flat=True) + + # If an image was uploaded, update the data to point to the new image. + image_url = upload_image_file(request.files.get('image')) + + if image_url: + data['imageUrl'] = image_url + + book = firestore.update(data, book_id) + + return redirect(url_for('.view', book_id=book['id'])) + + return render_template('form.html', action='Edit', book=book) + + +@app.route('/books//delete') +def delete(book_id): + firestore.delete(book_id) + return redirect(url_for('.list')) + + +@app.route('/logs') +def logs(): + logging.info('Hey, you triggered a custom log entry. Good job!') + flash(Markup('''You triggered a custom log entry. You can view it in the + Cloud Console''')) + return redirect(url_for('.list')) + + +@app.route('/errors') +def errors(): + raise Exception('This is an intentional exception.') + + +# Add an error handler that reports exceptions to Stackdriver Error +# Reporting. Note that this error handler is only used when debug +# is False +@app.errorhandler(500) +def server_error(e): + client = error_reporting.Client() + client.report_exception( + http_context=error_reporting.build_flask_context(request)) + return """ + An internal error occurred:
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +# This is only used when running locally. When running live, gunicorn runs +# the application. +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/bookshelf/main_test.py b/bookshelf/main_test.py new file mode 100644 index 00000000..249cba17 --- /dev/null +++ b/bookshelf/main_test.py @@ -0,0 +1,169 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import google.auth +import main +import pytest +import requests +from six import BytesIO + + +credentials, project_id = google.auth.default() +os.environ['GOOGLE_CLOUD_PROJECT'] = project_id + + +@pytest.fixture +def app(request): + """This fixture provides a Flask app instance configured for testing. + + It also ensures the tests run within a request context, allowing + any calls to flask.request, flask.current_app, etc. to work.""" + app = main.app + + with app.test_request_context(): + yield app + + +@pytest.fixture +def firestore(): + """This fixture provides a modified version of the app's Firebase model + that tracks all created items and deletes them at the end of the test. + + Any tests that directly or indirectly interact with the database should + use this to ensure that resources are properly cleaned up. + """ + + import firestore + + # Ensure no books exist before running the tests. This typically helps if + # tests somehow left the database in a bad state. + delete_all_books(firestore) + + yield firestore + + # Delete all books that we created during tests. + delete_all_books(firestore) + + +def delete_all_books(firestore): + while True: + books, _ = firestore.next_page(limit=50) + if not books: + break + for book in books: + firestore.delete(book['id']) + + +def test_list(app, firestore): + for i in range(1, 12): + firestore.create({'title': u'Book {0}'.format(i)}) + + with app.test_client() as c: + rv = c.get('/') + + assert rv.status == '200 OK' + + body = rv.data.decode('utf-8') + assert 'Book 1' in body, "Should show books" + assert len(re.findall('

Book', body)) <= 10, ( + "Should not show more than 10 books") + assert 'More' in body, "Should have more than one page" + + +def test_add(app): + data = { + 'title': 'Test Book', + 'author': 'Test Author', + 'publishedDate': 'Test Date Published', + 'description': 'Test Description' + } + + with app.test_client() as c: + rv = c.post('books/add', data=data, follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + assert 'Test Book' in body + assert 'Test Author' in body + assert 'Test Date Published' in body + assert 'Test Description' in body + + +def test_edit(app, firestore): + existing = firestore.create({'title': "Temp Title"}) + + with app.test_client() as c: + rv = c.post( + 'books/%s/edit' % existing['id'], + data={'title': 'Updated Title'}, + follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + assert 'Updated Title' in body + assert 'Temp Title' not in body + + +def test_delete(app, firestore): + existing = firestore.create({'title': "Temp Title"}) + + with app.test_client() as c: + rv = c.get( + 'books/%s/delete' % existing['id'], + follow_redirects=True) + + assert rv.status == '200 OK' + assert not firestore.read(existing['id']) + + +def test_upload_image(app): + data = { + 'title': 'Test Book', + 'author': 'Test Author', + 'publishedDate': 'Test Date Published', + 'description': 'Test Description', + 'image': (BytesIO(b'hello world'), 'hello.jpg') + } + + with app.test_client() as c: + rv = c.post('books/add', data=data, follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + + img_tag = re.search(''), + '1337h4x0r.php') + } + + with app.test_client() as c: + rv = c.post('/books/add', data=data, follow_redirects=True) + + # check we weren't pwned + assert rv.status == '400 BAD REQUEST' diff --git a/bookshelf/requirements.txt b/bookshelf/requirements.txt new file mode 100644 index 00000000..27d3d4fe --- /dev/null +++ b/bookshelf/requirements.txt @@ -0,0 +1,7 @@ +Flask==2.2.5 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.9.0 +google-cloud-error-reporting==1.9.1 +google-cloud-logging==3.5.0 +gunicorn==20.1.0 +six==1.16.0 diff --git a/5-logging/bookshelf/storage.py b/bookshelf/storage.py similarity index 73% rename from 5-logging/bookshelf/storage.py rename to bookshelf/storage.py index e8f523a4..f18b78df 100644 --- a/5-logging/bookshelf/storage.py +++ b/bookshelf/storage.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,30 +15,26 @@ from __future__ import absolute_import import datetime +import os from flask import current_app from google.cloud import storage import six -from werkzeug import secure_filename from werkzeug.exceptions import BadRequest - - -def _get_storage_client(): - return storage.Client( - project=current_app.config['PROJECT_ID']) +from werkzeug.utils import secure_filename def _check_extension(filename, allowed_extensions): - if ('.' not in filename or - filename.split('.').pop().lower() not in allowed_extensions): + file, ext = os.path.splitext(filename) + if (ext.replace('.', '') not in allowed_extensions): raise BadRequest( - "{0} has an invalid name or extension".format(filename)) + '{0} has an invalid name or extension'.format(filename)) def _safe_filename(filename): """ - Generates a safe filename that is unlikely to collide with existing objects - in Google Cloud Storage. + Generates a safe filename that is unlikely to collide with existing + objects in Google Cloud Storage. ``filename.ext`` is transformed into ``filename-YYYY-MM-DD-HHMMSS.ext`` """ @@ -56,15 +52,22 @@ def upload_file(file_stream, filename, content_type): _check_extension(filename, current_app.config['ALLOWED_EXTENSIONS']) filename = _safe_filename(filename) - client = _get_storage_client() - bucket = client.bucket(current_app.config['CLOUD_STORAGE_BUCKET']) + bucketname = os.getenv('GOOGLE_STORAGE_BUCKET') or os.getenv( + 'GOOGLE_CLOUD_PROJECT') + '_bucket' + + # [START bookshelf_cloud_storage_client] + client = storage.Client() + bucket = client.bucket(bucketname) blob = bucket.blob(filename) blob.upload_from_string( file_stream, content_type=content_type) + # Ensure the file is publicly readable. + blob.make_public() url = blob.public_url + # [END bookshelf_cloud_storage_client] if isinstance(url, six.binary_type): url = url.decode('utf-8') diff --git a/7-gce/bookshelf/templates/base.html b/bookshelf/templates/base.html similarity index 94% rename from 7-gce/bookshelf/templates/base.html rename to bookshelf/templates/base.html index a18c7908..0c273faf 100644 --- a/7-gce/bookshelf/templates/base.html +++ b/bookshelf/templates/base.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ diff --git a/3-binary-data/bookshelf/templates/form.html b/bookshelf/templates/form.html similarity index 98% rename from 3-binary-data/bookshelf/templates/form.html rename to bookshelf/templates/form.html index 6d5e4d1f..64950d2e 100644 --- a/3-binary-data/bookshelf/templates/form.html +++ b/bookshelf/templates/form.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/6-pubsub/bookshelf/templates/list.html b/bookshelf/templates/list.html similarity index 80% rename from 6-pubsub/bookshelf/templates/list.html rename to bookshelf/templates/list.html index 3362f0e2..b255d209 100644 --- a/6-pubsub/bookshelf/templates/list.html +++ b/bookshelf/templates/list.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,12 @@ {% block content %} +{% with messages = get_flashed_messages() %} + {% for message in messages %} +

{{ message }}

+ {% endfor %} +{% endwith %} +

Books

@@ -31,7 +37,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
@@ -44,10 +50,10 @@

{{book.title}}

No books found

{% endfor %} -{% if next_page_token %} +{% if last_title %}
{% endif %} diff --git a/3-binary-data/bookshelf/templates/view.html b/bookshelf/templates/view.html similarity index 93% rename from 3-binary-data/bookshelf/templates/view.html rename to bookshelf/templates/view.html index 1e509ff0..476db111 100644 --- a/3-binary-data/bookshelf/templates/view.html +++ b/bookshelf/templates/view.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
{# [END book_image] #} diff --git a/conftest.py b/conftest.py deleted file mode 100644 index b6078c07..00000000 --- a/conftest.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import mock -import pytest - - -@pytest.fixture -def api_client_inject_project_id(): - """Patches all googleapiclient requests to replace 'YOUR_PROJECT_ID' with - the project ID.""" - import googleapiclient.http - - old_execute = googleapiclient.http.HttpRequest.execute - - def new_execute(self, http=None, num_retries=0): - self.uri = self.uri.replace('YOUR_PROJECT_ID', PROJECT) - return old_execute(self, http=http, num_retries=num_retries) - - with mock.patch( - 'googleapiclient.http.HttpRequest.execute', - new=new_execute): - yield diff --git a/decrypt-secrets.sh b/decrypt-secrets.sh index 991919ce..7df5428a 100755 --- a/decrypt-secrets.sh +++ b/decrypt-secrets.sh @@ -14,7 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -password=$1 +set -euo pipefail -openssl aes-256-cbc -k "$password" -in secrets.tar.enc -out secrets.tar -d -tar xvf secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +# If there's already a secret file, skip retrieving the secret. +if [[ -f "service-account.json" ]]; then + echo "The secret already exists, skipping." + exit 0 +fi + +gcloud secrets versions access latest \ + --secret="getting-started-python-service-account" \ + --project="${project_id}" \ + > service-account.json diff --git a/encrypt-secrets.sh b/encrypt-secrets.sh index 10f408bb..ddad76db 100755 --- a/encrypt-secrets.sh +++ b/encrypt-secrets.sh @@ -14,11 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -read -s -p "Enter password for encryption: " password -echo +set -euo pipefail -tar cvf secrets.tar {service-account.json,config.py} -openssl aes-256-cbc -k "$password" -in secrets.tar -out secrets.tar.enc -rm secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} -travis encrypt "secrets_password=$password" --add +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions add "getting-started-python-service-account" \ + --project="${project_id}" \ + --data-file="service-account.json" diff --git a/gce/README.md b/gce/README.md new file mode 100644 index 00000000..d7c9f1ff --- /dev/null +++ b/gce/README.md @@ -0,0 +1,7 @@ +# Hello World for Python on Google Compute Engine + + This folder contains the sample code for the [Deploying to Google Compute Engine][tutorial-gce] +tutorial. Please refer to the tutorial for instructions on configuring, running, +and deploying this sample. + + [tutorial-gce]: https://cloud.google.com/python/tutorials/getting-started-on-compute-engine diff --git a/gce/add-google-cloud-ops-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh new file mode 100644 index 00000000..63cf2dbd --- /dev/null +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -0,0 +1,519 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *NOTE*: The source of truth for this script is: +# https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +# See https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/installation +# for installation instructions. +# It is committed to this repository to follow security best practices. +# +# +# Add repository for the Google ops agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--uninstall-standalone-logging-agent`: +# Uninstalls the standalone logging agent (`google-fluentd`). +# +# * `--uninstall-standalone-monitoring-agent`: +# Uninstalls the standalone monitoring agent (`stackdriver-agent`). +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-google-cloud-ops-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --dry-run +# +# * To replace standalone agents with the Ops agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --uninstall-standalone-logging-agent --uninstall-standalone-monitoring-agent +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-ops-agent-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + uninstall-standalone-logging-agent) ACTIONS+=('uninstall-standalone-logging-agent') ;; + uninstall-standalone-monitoring-agent) ACTIONS+=('uninstall-standalone-monitoring-agent') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ " ${ACTIONS[*]} " == *\ uninstall\ * || ( " ${ACTIONS[*]} " == *\ remove-repo\ * && " ${ACTIONS[*]} " != *\ also-install\ * )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE + +if [[ " ${ACTIONS[*]} " == *\ also-install*uninstall\ * ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the ops agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/stackdriver/docs/solutions/ops-agent' + +# URL documentation which lists supported platforms for running the ops agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#supported_operating_systems" + +# Packages to install. +AGENT_PACKAGE='google-cloud-ops-agent' +declare -a ADDITIONAL_PACKAGES=() + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-ops-agent ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-ops-agent.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-ops-agent.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-ops-agent.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-ops-agent.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 2.0.1~debian9.13. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}~*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early unless at least one package is installed. + dpkg -l "${packages[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-ops-agent.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-ops-agent/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-ops-agent || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +save_configuration_files() { + local save_dir="/var/lib/google-cloud-ops-agent/saved_configs" + ${DRY_RUN} mkdir -p "${save_dir}" + ${DRY_RUN} cp -rp "$@" "${save_dir}" + echo "$* is now copied over to ${save_dir} folder." +} + +main() { + case "${ID:-}" in + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-logging-agent\ * ]]; then + save_configuration_files "/etc/google-fluentd" + # This will also remove dependent packages, e.g. "google-fluentd-catch-all-config" or "google-fluentd-catch-all-config-structured". + uninstall "google-fluentd" + fi + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-monitoring-agent\ * ]]; then + save_configuration_files "/etc/stackdriver" "/opt/stackdriver/collectd/etc" + uninstall "stackdriver-agent" + fi + if [[ " ${ACTIONS[*]} " == *\ add-repo\ * ]]; then + resolve_version + add_repo + fi + if [[ " ${ACTIONS[*]} " == *\ also-install\ * ]]; then + install_agent + elif [[ " ${ACTIONS[*]} " == *\ uninstall\ * ]]; then + save_configuration_files "/etc/google-cloud-ops-agent" + uninstall "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" + fi + if [[ " ${ACTIONS[*]} " == *\ remove-repo\ * ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/deploy.sh b/gce/deploy.sh new file mode 100644 index 00000000..36ed75ca --- /dev/null +++ b/gce/deploy.sh @@ -0,0 +1,35 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# [START getting_started_gce_create_instance] +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a + +gcloud compute instances create $MY_INSTANCE_NAME \ + --image-family=debian-10 \ + --image-project=debian-cloud \ + --machine-type=g1-small \ + --scopes userinfo-email,cloud-platform \ + --metadata-from-file startup-script=startup-script.sh \ + --zone $ZONE \ + --tags http-server +# [END getting_started_gce_create_instance] + +gcloud compute firewall-rules create default-allow-http-8080 \ + --allow tcp:8080 \ + --source-ranges 0.0.0.0/0 \ + --target-tags http-server \ + --description "Allow port 8080 access to http-server" diff --git a/3-binary-data/main.py b/gce/main.py similarity index 76% rename from 3-binary-data/main.py rename to gce/main.py index d5697c6c..f4435efc 100644 --- a/3-binary-data/main.py +++ b/gce/main.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config +from flask import Flask +app = Flask(__name__) -app = bookshelf.create_app(config) +@app.route('/', methods=['GET']) +def say_hello(): + return "Hello, world!" -# This is only used when running locally. When running live, gunicorn runs -# the application. if __name__ == '__main__': app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/4-auth/main.py b/gce/main_test.py similarity index 67% rename from 4-auth/main.py rename to gce/main_test.py index d5697c6c..6d1f887b 100644 --- a/4-auth/main.py +++ b/gce/main_test.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config +import main -app = bookshelf.create_app(config) +def test_hello(): + main.app.testing = True + client = main.app.test_client() - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Hello, world!" in response_text diff --git a/gce/procfile b/gce/procfile new file mode 100644 index 00000000..9d9842b4 --- /dev/null +++ b/gce/procfile @@ -0,0 +1 @@ +hello: /opt/app/gce/env/bin/gunicorn -b 0.0.0.0:8080 main:app diff --git a/gce/python-app.conf b/gce/python-app.conf new file mode 100644 index 00000000..d3a9b17b --- /dev/null +++ b/gce/python-app.conf @@ -0,0 +1,11 @@ +[program:pythonapp] +directory=/opt/app/gce +command=/opt/app/gce/env/bin/honcho start -f ./procfile hello +autostart=true +autorestart=true +user=pythonapp +# Environment variables ensure that the application runs inside of the +# configured virtualenv. +environment=VIRTUAL_ENV="/opt/app/gce/env",PATH="/opt/app/gce/env/bin",HOME="/home/pythonapp",USER="pythonapp" +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/gce/requirements.txt b/gce/requirements.txt new file mode 100644 index 00000000..b655465f --- /dev/null +++ b/gce/requirements.txt @@ -0,0 +1,3 @@ +flask==2.2.5 +honcho==1.1.0 +gunicorn==20.1.0 diff --git a/gce/startup-script.sh b/gce/startup-script.sh new file mode 100644 index 00000000..cc35d30f --- /dev/null +++ b/gce/startup-script.sh @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Echo commands and fail on error +set -ev + +# [START getting_started_gce_startup_script] +# Install or update needed software +apt-get update +apt-get install -yq git supervisor python python-pip python3-distutils +pip install --upgrade pip virtualenv + +# Fetch source code +export HOME=/root +git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app + +# Install Cloud Ops Agent +sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install + +# Account to own server process +useradd -m -d /home/pythonapp pythonapp + +# Python environment setup +virtualenv -p python3 /opt/app/gce/env +/bin/bash -c "source /opt/app/gce/env/bin/activate" +/opt/app/gce/env/bin/pip install -r /opt/app/gce/requirements.txt + +# Set ownership to newly created account +chown -R pythonapp:pythonapp /opt/app + +# Put supervisor configuration in proper place +cp /opt/app/gce/python-app.conf /etc/supervisor/conf.d/python-app.conf + +# Start service via supervisorctl +supervisorctl reread +supervisorctl update +# [END getting_started_gce_startup_script] diff --git a/1-hello-world/main.py b/gce/teardown.sh similarity index 66% rename from 1-hello-world/main.py rename to gce/teardown.sh index 5b398d30..668b1218 100644 --- a/1-hello-world/main.py +++ b/gce/teardown.sh @@ -1,4 +1,6 @@ -# Copyright 2015 Google Inc. + #! /bin/bash + +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START app] -from flask import Flask - - -app = Flask(__name__) - +set -x -@app.route('/') -def hello(): - """Return a friendly HTTP greeting.""" - return 'Hello World!' +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a +gcloud compute instances delete $MY_INSTANCE_NAME \ + --zone=$ZONE --delete-disks=all -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080) -# [END app] +gcloud compute firewall-rules delete default-allow-http-8080 diff --git a/noxfile.py b/noxfile.py index 35d37061..8f5a921c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,15 +7,12 @@ 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' DIRS = [ - # Hello world doesn't have system tests, just a lint test which will be - # covered by the global lint here. - # '1-hello-world', - '2-structured-data', - '3-binary-data', - '4-auth', - '5-logging', - '6-pubsub', - '7-gce', + 'authenticating-users', + 'background/app', + 'background/function', + 'gce', + 'sessions', + 'bookshelf', ] PYTEST_COMMON_ARGS = ['--junitxml=sponge_log.xml', '-m', 'not e2e'] @@ -53,22 +50,13 @@ def run_test(session, dir): 'pytest', *(PYTEST_COMMON_ARGS + session.posargs), # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. + # when slow and flaky tests are excluded. # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html success_codes=[0, 5]) -@nox.session +@nox.session(python="3.12") @nox.parametrize('dir', DIRS) def run_tests(session, dir=None): """Run all tests for all directories (slow!)""" run_test(session, dir) - - -@nox.session -@nox.parametrize('dir', DIRS) -def travis(session, dir=None): - """On travis, only run the py3.4 and cloudsql tests.""" - run_tests( - session, - dir=dir) diff --git a/optional-kubernetes-engine/.dockerignore b/optional-kubernetes-engine/.dockerignore index fdc976d4..c4730648 100644 --- a/optional-kubernetes-engine/.dockerignore +++ b/optional-kubernetes-engine/.dockerignore @@ -12,6 +12,9 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log .git +.mypy_cache +.pytest_cache +.hypothesis diff --git a/optional-kubernetes-engine/Makefile b/optional-kubernetes-engine/Makefile index f275ddf4..4c1cbf3e 100644 --- a/optional-kubernetes-engine/Makefile +++ b/optional-kubernetes-engine/Makefile @@ -12,8 +12,8 @@ create-cluster: .PHONY: create-bucket create-bucket: - gsutil mb gs://$(GCLOUD_PROJECT) - gsutil defacl set public-read gs://$(GCLOUD_PROJECT) + gcloud storage buckets create gs://$(GCLOUD_PROJECT) + gcloud storage buckets update gs://$(GCLOUD_PROJECT) --predefined-default-object-acl=public-read .PHONY: build build: diff --git a/optional-kubernetes-engine/README.md b/optional-kubernetes-engine/README.md index e3c17236..9acb98b4 100644 --- a/optional-kubernetes-engine/README.md +++ b/optional-kubernetes-engine/README.md @@ -36,8 +36,8 @@ Alternatively, you can use make: The bookshelf application uses [Google Cloud Storage](https://cloud.google.com/storage) to store image files. Create a bucket for your project: - gsutil mb gs:// - gsutil defacl set public-read gs:// + gcloud storage buckets create gs:// + gcloud storage buckets update gs:// --predefined-default-object-acl=public-read Alternatively, you can use make: diff --git a/optional-kubernetes-engine/bookshelf-frontend.yaml b/optional-kubernetes-engine/bookshelf-frontend.yaml index 265c8e35..5db2d638 100644 --- a/optional-kubernetes-engine/bookshelf-frontend.yaml +++ b/optional-kubernetes-engine/bookshelf-frontend.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf application frontend. The frontend serves # public web traffic. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-frontend @@ -27,6 +27,10 @@ metadata: # https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 3 + selector: + matchLabels: + app: bookshelf + tier: frontend template: metadata: labels: diff --git a/optional-kubernetes-engine/bookshelf-worker.yaml b/optional-kubernetes-engine/bookshelf-worker.yaml index c30df89b..979437b6 100644 --- a/optional-kubernetes-engine/bookshelf-worker.yaml +++ b/optional-kubernetes-engine/bookshelf-worker.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf task worker. The worker is responsible # for processing book requests and updating book information. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-worker @@ -27,6 +27,10 @@ metadata: # https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 2 + selector: + matchLabels: + app: bookshelf + tier: worker template: metadata: labels: diff --git a/optional-kubernetes-engine/bookshelf/templates/list.html b/optional-kubernetes-engine/bookshelf/templates/list.html index 3362f0e2..a80d5b56 100644 --- a/optional-kubernetes-engine/bookshelf/templates/list.html +++ b/optional-kubernetes-engine/bookshelf/templates/list.html @@ -31,7 +31,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/optional-kubernetes-engine/bookshelf/templates/view.html b/optional-kubernetes-engine/bookshelf/templates/view.html index e654e8ab..2f337229 100644 --- a/optional-kubernetes-engine/bookshelf/templates/view.html +++ b/optional-kubernetes-engine/bookshelf/templates/view.html @@ -36,7 +36,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/optional-kubernetes-engine/config.py b/optional-kubernetes-engine/config.py index f77c8c75..7da9b4cc 100644 --- a/optional-kubernetes-engine/config.py +++ b/optional-kubernetes-engine/config.py @@ -78,16 +78,16 @@ # Typically, you'll name your bucket the same as your project. To create a # bucket: # -# $ gsutil mb gs:// +# $ gcloud storage buckets create gs:// # # You also need to make sure that the default ACL is set to public-read, # otherwise users will not be able to see their upload images: # -# $ gsutil defacl set public-read gs:// +# $ gcloud storage buckets update --predefined-default-object-acl=public-read gs:// # # You can adjust the max content length and allow extensions settings to allow # larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-project-id' +CLOUD_STORAGE_BUCKET = 'your-bucket-name' MAX_CONTENT_LENGTH = 8 * 1024 * 1024 ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) diff --git a/optional-kubernetes-engine/requirements.txt b/optional-kubernetes-engine/requirements.txt index 5d998499..33ba8279 100644 --- a/optional-kubernetes-engine/requirements.txt +++ b/optional-kubernetes-engine/requirements.txt @@ -1,6 +1,6 @@ -Flask>=1.0.0 +Flask==1.0.4 google-cloud-datastore==1.7.1 -google-cloud-storage==1.13.0 +google-cloud-storage==1.23.0 google-cloud-logging==1.8.0 google-cloud-error_reporting==0.30.0 gunicorn==19.9.0 @@ -8,9 +8,9 @@ oauth2client==4.1.3 mock==2.0.0 Flask-SQLAlchemy==2.3.2 PyMySQL==0.9.2 -Flask-PyMongo>=2.0.0 +Flask-PyMongo==2.3.0 PyMongo==3.7.2 six==1.11.0 -requests[security]==2.20.1 +requests[security]==2.21.0 honcho==1.0.1 psq==0.7.0 diff --git a/optional-kubernetes-engine/tests/conftest.py b/optional-kubernetes-engine/tests/conftest.py index 8123575b..57a50a1c 100644 --- a/optional-kubernetes-engine/tests/conftest.py +++ b/optional-kubernetes-engine/tests/conftest.py @@ -22,7 +22,7 @@ from retrying import retry -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) +@pytest.fixture(params=['datastore', 'mongodb']) def app(request): """This fixtures provides a Flask app instance configured for testing. @@ -43,7 +43,7 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def model(monkeypatch, app): """This fixture provides a modified version of the app's model that tracks all created items and deletes them at the end of the test. diff --git a/optional-kubernetes-engine/tests/test_auth.py b/optional-kubernetes-engine/tests/test_auth.py index 491de9e0..6367608c 100644 --- a/optional-kubernetes-engine/tests/test_auth.py +++ b/optional-kubernetes-engine/tests/test_auth.py @@ -41,7 +41,10 @@ def client_with_credentials(app): def inner(): with app.test_client() as client: with client.session_transaction() as session: - session['profile'] = {'email': 'abc@example.com', 'name': 'Test User'} + session['profile'] = { + 'email': 'abc@example.com', + 'name': 'Test User' + } session['google_oauth2_credentials'] = credentials.to_json() yield client diff --git a/pytest.ini b/pytest.ini index 2008cdfa..eb509ac0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,5 @@ [pytest] addopts = -v - --no-success-flaky-report --tb=native norecursedirs = .git env lib .tox .nox diff --git a/requirements.txt b/requirements.txt index 4b0a8521..0fd66f54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ -BeautifulSoup4==4.6.0 -flake8==3.5.0 -flaky==3.4.0 -mock==2.0.0 -nox>=2018.10.9 -pytest==3.8.2 -pytest-cov==2.5.1 -retrying==1.3.3 -requests>=2.20.0 +flake8===5.0.4; python_version < '3.8' +flake8==6.0.0; python_version >= '3.8' +pytest==7.3.1 +nox==2023.4.22 +requests==2.31.0 diff --git a/secrets.tar.enc b/secrets.tar.enc index 2942865e..25ccbe61 100644 Binary files a/secrets.tar.enc and b/secrets.tar.enc differ diff --git a/sessions/app.yaml b/sessions/app.yaml new file mode 100644 index 00000000..0b2d0e28 --- /dev/null +++ b/sessions/app.yaml @@ -0,0 +1,17 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_sessions_runtime] +runtime: python37 +# [END getting_started_sessions_runtime] diff --git a/sessions/main.py b/sessions/main.py new file mode 100644 index 00000000..62871ba1 --- /dev/null +++ b/sessions/main.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_sessions_all] +import random +from uuid import uuid4 + +from flask import Flask, make_response, request +from google.cloud import firestore + + +app = Flask(__name__) +db = firestore.Client() +sessions = db.collection('sessions') +greetings = [ + 'Hello World', + 'Hallo Welt', + 'Ciao Mondo', + 'Salut le Monde', + 'Hola Mundo', +] + + +@firestore.transactional +def get_session_data(transaction, session_id): + """ Looks up (or creates) the session with the given session_id. + Creates a random session_id if none is provided. Increments + the number of views in this session. Updates are done in a + transaction to make sure no saved increments are overwritten. + """ + if session_id is None: + session_id = str(uuid4()) # Random, unique identifier + + doc_ref = sessions.document(document_id=session_id) + doc = doc_ref.get(transaction=transaction) + if doc.exists: + session = doc.to_dict() + else: + session = { + 'greeting': random.choice(greetings), + 'views': 0 + } + + session['views'] += 1 # This counts as a view + transaction.set(doc_ref, session) + + session['session_id'] = session_id + return session + + +@app.route('/', methods=['GET']) +def home(): + template = '{} views for "{}"' + + transaction = db.transaction() + session = get_session_data(transaction, request.cookies.get('session_id')) + + resp = make_response(template.format( + session['views'], + session['greeting'] + ) + ) + resp.set_cookie('session_id', session['session_id'], httponly=True) + return resp + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080) +# [END getting_started_sessions_all] diff --git a/sessions/main_test.py b/sessions/main_test.py new file mode 100644 index 00000000..06965c08 --- /dev/null +++ b/sessions/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import uuid + +import main +import pytest + + +@pytest.fixture +def client(): + """ Yields a test client, AND creates and later cleans up a + dummy collection for sessions. + """ + main.app.testing = True + + # Override the Firestore collection used for sessions in main + main.sessions = main.db.collection(str(uuid.uuid4())) + + client = main.app.test_client() + yield client + + # Clean up session objects created in test collection + for doc_ref in main.sessions.list_documents(): + doc_ref.delete() + + +def test_session(client): + r = client.get('/') + assert r.status_code == 200 + data = r.data.decode('utf-8') + assert '1 views' in data + + match = re.search('views for "([A-Za-z ]+)"', data) + assert match is not None + greeting = match.group(1) + + r = client.get('/') + assert r.status_code == 200 + data = r.data.decode('utf-8') + assert '2 views' in data + assert greeting in data diff --git a/sessions/requirements-dev.txt b/sessions/requirements-dev.txt new file mode 100644 index 00000000..fe969938 --- /dev/null +++ b/sessions/requirements-dev.txt @@ -0,0 +1 @@ +pytest>=5.0.0 diff --git a/sessions/requirements.txt b/sessions/requirements.txt new file mode 100644 index 00000000..e0e267b5 --- /dev/null +++ b/sessions/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-firestore==2.11.1 +flask==2.2.5