diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..c0a95e72 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [ + "optional-kubernetes-engine" + ] +} diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1 @@ + diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 00000000..7d8eca8e --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,40 @@ +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: true + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: false + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: true + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "kokoro" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: + # Team slug to add to repository permissions + - team: yoshi-admins + # Access level required, one of push|pull|admin + permission: admin + - team: python-samples-reviewers + permission: admin diff --git a/.gitignore b/.gitignore index 6f2c88b4..1e32d951 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,9 @@ htmlcov/ .cache nosetests.xml coverage.xml +*_log.xml *,cover +sponge_log.xml *.log diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg new file mode 100644 index 00000000..f58e4f76 --- /dev/null +++ b/.kokoro/common.cfg @@ -0,0 +1,28 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download trampoline resources. These will be in ${KOKORO_GFILE_DIR} +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/getting-started-python" + +# All builds use the trampoline script to run in docker. +build_file: "getting-started-python/.kokoro/trampoline_v2.sh" + +# Use the Python worker docker iamge. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python/getting-started-python" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: ".kokoro/system_tests.sh" +} + +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile new file mode 100644 index 00000000..accdd0bf --- /dev/null +++ b/.kokoro/docker/Dockerfile @@ -0,0 +1,57 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-devrel-kokoro-resources/python-base:latest + +# Install libraries needed by third-party python packages that we depend on. +RUN apt-get update \ + && apt-get install -y \ + graphviz \ + libcurl4-openssl-dev \ + libffi-dev \ + libjpeg-dev \ + libmagickwand-dev \ + libmemcached-dev \ + libmysqlclient-dev \ + libpng-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + openssl \ + zlib1g-dev \ + && apt-get clean + + +###################### Check python version + +RUN python3 --version +RUN which python3 + +# Setup Cloud SDK +ENV CLOUD_SDK_VERSION 489.0.0 +# Use system python for cloud sdk. +ENV CLOUDSDK_PYTHON python3.12 +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN /google-cloud-sdk/install.sh +ENV PATH /google-cloud-sdk/bin:$PATH + +# Setup the user profile for pip +ENV PATH ~/.local/bin:/root/.local/bin:$PATH + +# Install the current version of nox. +RUN python3 -m pip install --user --no-cache-dir nox==2022.1.7 + +CMD ["nox"] diff --git a/.kokoro/presubmit.cfg b/.kokoro/presubmit.cfg new file mode 100644 index 00000000..e69de29b diff --git a/.kokoro/system_tests.cfg b/.kokoro/system_tests.cfg new file mode 100644 index 00000000..e69de29b diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh new file mode 100755 index 00000000..29bb4ae5 --- /dev/null +++ b/.kokoro/system_tests.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin + +cd "${PROJECT_ROOT:-github/getting-started-python}" + + +# Unencrypt and extract secrets +SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") +./decrypt-secrets.sh "${SECRETS_PASSWORD}" + +# Setup environment variables +export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service-account.json" + +# This block is executed only with Trampoline V2. +if [[ -n "${TRAMPOLINE_VERSION:-}" ]]; then + # Install nox as a user and add it to the PATH. + python3 -m pip install --user nox + export PATH="${PATH}:${HOME}/.local/bin" +fi + +# Run tests +nox -s lint +nox -s run_tests + +# If this is a nightly build, send the test log to the Flaky Bot. +# See https://github.com/googleapis/repo-automation-bots/tree/HEAD/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"system_tests"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot +fi diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh new file mode 100755 index 00000000..fef7c24d --- /dev/null +++ b/.kokoro/trampoline.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. + +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..ef6972b4 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.10" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Flaky Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + "KOKORO_BUILD_ARTIFACTS_SUBDIR" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..17f21195 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + # We test this envvar in tests/python/test_envvar.py. + "TEST_ENV" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# The build will show some commands and docker build logs. +TRAMPOLINE_VERBOSE="true" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 02e0aadb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -sudo: false -language: python -env: - global: - - GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/service-account.json - - secure: S/A6XENZTy+py6kQxUX5EWgNJv3NGRaSTo4LdWVaZ7qKww1+akeMNHKvPXjp398UtsXkCWCpdy5HeLDYdmzX83uuFoPD1S/z7ilVpo/q4sRxDQmnw4diPHSpx32+XRyKpMpbjgJTMhocqZGcJFI9jxZ4QAHK3rul8ulvTojCyQf7OrePhgfBB72f/ZHho73VqmsngGyWNI6MtGGrBzMTtXlAOt8BgDhYXzPjhEWkE+OKAF9sMJqFmxUXyKkfTBfcGHDYcAOlNtvVdZGRC9NmzRvZxC5T7lQv/WdfGBAT3Zr4NSobKHK6DAv9Q7QvVgHIs7HRomgpHQ9IKnwyhR4RTYAnTtZyHe0/fUlPUXC8+Pkk634eRgYuVNMH8Cf7eRXcLRGF6wgBe9nHnx/o0w/kLbEyirQo3kVIM4Y6yfR5MhZ0Nb4oRq5tuXzqN1B5ZFyfmHhkwsJoeL5wgIGcwOxkoCTH8HqvPfW+SJuUpQC/sB89ixfTfUXYyMJxgGTOdg+1wD4IAfivGyWiUBcx06glfC3Sn+75oflgMz4M8y6zafXTzux755U5pEMa8Gw+5BqpDlkRliZwdAJ6UwHo/XHeJ2rgiTddEgSZYN0CR5XQFcEc69lauUEPkeE5fPubSgcH28xvbhWq7WJ1qdi3Wdw8RaCLOys/B6vwL8GhSJP+ROg= -before_install: -- ./decrypt-secrets.sh "$SECRETS_PASSWORD" -install: -- pip install nox-automation tox -script: -- nox --session lint travis diff --git a/1-hello-world/app.yaml b/1-hello-world/app.yaml deleted file mode 100644 index 4e3cb73b..00000000 --- a/1-hello-world/app.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -# [START runtime] -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 -# [END runtime] diff --git a/1-hello-world/requirements.txt b/1-hello-world/requirements.txt deleted file mode 100644 index 45c7016a..00000000 --- a/1-hello-world/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Flask==0.11.1 -gunicorn==19.6.0 diff --git a/1-hello-world/tox.ini b/1-hello-world/tox.ini deleted file mode 100644 index 82bf72e3..00000000 --- a/1-hello-world/tox.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -skipsdist = True -envlist = lint - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --exclude=env --import-order-style=google diff --git a/2-structured-data/app.yaml b/2-structured-data/app.yaml deleted file mode 100644 index 566e6a19..00000000 --- a/2-structured-data/app.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -#[START cloudsql_settings] -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" -#[END cloudsql_settings] diff --git a/2-structured-data/bookshelf/__init__.py b/2-structured-data/bookshelf/__init__.py deleted file mode 100644 index 419c524c..00000000 --- a/2-structured-data/bookshelf/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from flask import current_app, Flask, redirect, url_for - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model diff --git a/2-structured-data/bookshelf/crud.py b/2-structured-data/bookshelf/crud.py deleted file mode 100644 index baa3488d..00000000 --- a/2-structured-data/bookshelf/crud.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model -from flask import Blueprint, redirect, render_template, request, url_for - - -crud = Blueprint('crud', __name__) - - -# [START list] -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) -# [END list] - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -# [START add] -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) -# [END add] - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/2-structured-data/bookshelf/model_cloudsql.py b/2-structured-data/bookshelf/model_cloudsql.py deleted file mode 100644 index 39ab1bbb..00000000 --- a/2-structured-data/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -# [START model] -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return "{{action}} book - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - -
- -{% endblock %} -{# [END form] #} diff --git a/2-structured-data/bookshelf/templates/list.html b/2-structured-data/bookshelf/templates/list.html deleted file mode 100644 index 69beeeb3..00000000 --- a/2-structured-data/bookshelf/templates/list.html +++ /dev/null @@ -1,51 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/2-structured-data/bookshelf/templates/view.html b/2-structured-data/bookshelf/templates/view.html deleted file mode 100644 index 5f07c51a..00000000 --- a/2-structured-data/bookshelf/templates/view.html +++ /dev/null @@ -1,48 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

-
-
- -{% endblock %} diff --git a/2-structured-data/config.py b/2-structured-data/config.py deleted file mode 100644 index 660387d3..00000000 --- a/2-structured-data/config.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = \ - 'mongodb://user:password@host:27017/database' diff --git a/2-structured-data/requirements-dev.txt b/2-structured-data/requirements-dev.txt deleted file mode 100644 index a86f7590..00000000 --- a/2-structured-data/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -pytest==3.0.5 -pytest-cov==2.4.0 -retrying==1.3.3 diff --git a/2-structured-data/requirements.txt b/2-structured-data/requirements.txt deleted file mode 100644 index 03f56a80..00000000 --- a/2-structured-data/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 diff --git a/2-structured-data/tests/conftest.py b/2-structured-data/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/2-structured-data/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/2-structured-data/tox.ini b/2-structured-data/tox.ini deleted file mode 100644 index 9c9bf102..00000000 --- a/2-structured-data/tox.ini +++ /dev/null @@ -1,23 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/3-binary-data/app.yaml b/3-binary-data/app.yaml deleted file mode 100644 index d0de9391..00000000 --- a/3-binary-data/app.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/config for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/3-binary-data/bookshelf/__init__.py b/3-binary-data/bookshelf/__init__.py deleted file mode 100644 index 419c524c..00000000 --- a/3-binary-data/bookshelf/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from flask import current_app, Flask, redirect, url_for - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model diff --git a/3-binary-data/bookshelf/crud.py b/3-binary-data/bookshelf/crud.py deleted file mode 100644 index 6686798c..00000000 --- a/3-binary-data/bookshelf/crud.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - url_for - - -crud = Blueprint('crud', __name__) - - -# [START upload_image_file] -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url -# [END upload_image_file] - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - # [START image_url] - image_url = upload_image_file(request.files.get('image')) - # [END image_url] - - # [START image_url2] - if image_url: - data['imageUrl'] = image_url - # [END image_url2] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/3-binary-data/bookshelf/model_cloudsql.py b/3-binary-data/bookshelf/model_cloudsql.py deleted file mode 100644 index 96eb190a..00000000 --- a/3-binary-data/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return "Books - - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/3-binary-data/config.py b/3-binary-data/config.py deleted file mode 100644 index 7d397967..00000000 --- a/3-binary-data/config.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) diff --git a/3-binary-data/requirements-dev.txt b/3-binary-data/requirements-dev.txt deleted file mode 100644 index a86f7590..00000000 --- a/3-binary-data/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -pytest==3.0.5 -pytest-cov==2.4.0 -retrying==1.3.3 diff --git a/3-binary-data/requirements.txt b/3-binary-data/requirements.txt deleted file mode 100644 index 03f56a80..00000000 --- a/3-binary-data/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 diff --git a/3-binary-data/tests/test_crud.py b/3-binary-data/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/3-binary-data/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/3-binary-data/tox.ini b/3-binary-data/tox.ini deleted file mode 100644 index 9c9bf102..00000000 --- a/3-binary-data/tox.ini +++ /dev/null @@ -1,23 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs} tests -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/4-auth/app.yaml b/4-auth/app.yaml deleted file mode 100644 index aa5ddf46..00000000 --- a/4-auth/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/4-auth/bookshelf/__init__.py b/4-auth/bookshelf/__init__.py deleted file mode 100644 index 3914ca63..00000000 --- a/4-auth/bookshelf/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -import httplib2 - -# [START include] -from oauth2client.contrib.flask_util import UserOAuth2 - -oauth2 = UserOAuth2() -# [END include] - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - logging.basicConfig(level=logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # [START init_app] - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - # [END init_app] - - # [START logout] - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - # [END logout] - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler. This is useful for debugging the live application, - # however, you should disable the output of the exception for production - # applications. - @app.errorhandler(500) - def server_error(e): - return """ - An internal error occurred:
{}
- See logs for full stacktrace. - """.format(e), 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -# [START request_user_info] -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google+ API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/plus/v1/people/me') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: %s" % resp) - return None - session['profile'] = json.loads(content.decode('utf-8')) - -# [END request_user_info] diff --git a/4-auth/bookshelf/crud.py b/4-auth/bookshelf/crud.py deleted file mode 100644 index 8991bcfe..00000000 --- a/4-auth/bookshelf/crud.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -# [START list_mine] -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['id'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) -# [END list_mine] - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -# [START add] -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['displayName'] - data['createdById'] = session['profile']['id'] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) -# [END add] - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/4-auth/bookshelf/model_cloudsql.py b/4-auth/bookshelf/model_cloudsql.py deleted file mode 100644 index c9456fcd..00000000 --- a/4-auth/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/4-auth/bookshelf/templates/view.html b/4-auth/bookshelf/templates/view.html deleted file mode 100644 index cfa9138e..00000000 --- a/4-auth/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.get('createdBy')|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/4-auth/config.py b/4-auth/config.py deleted file mode 100644 index 767efbd1..00000000 --- a/4-auth/config.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -# [START secret_key] -SECRET_KEY = 'secret' -# [END secret_key] - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/4-auth/requirements-dev.txt b/4-auth/requirements-dev.txt deleted file mode 100644 index cb5c475b..00000000 --- a/4-auth/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -pytest==3.0.5 -pytest-cov==2.4.0 -retrying==1.3.3 -mock==2.0.0 diff --git a/4-auth/requirements.txt b/4-auth/requirements.txt deleted file mode 100644 index 03f56a80..00000000 --- a/4-auth/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 diff --git a/4-auth/tests/conftest.py b/4-auth/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/4-auth/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/4-auth/tests/test_auth.py b/4-auth/tests/test_auth.py deleted file mode 100644 index a03e9440..00000000 --- a/4-auth/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'id': 'abc', 'displayName': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/4-auth/tests/test_crud.py b/4-auth/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/4-auth/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/4-auth/tests/test_storage.py b/4-auth/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/4-auth/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/4-auth/tox.ini b/4-auth/tox.ini deleted file mode 100644 index 680033e1..00000000 --- a/4-auth/tox.ini +++ /dev/null @@ -1,23 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs:tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/5-logging/app.yaml b/5-logging/app.yaml deleted file mode 100644 index aa5ddf46..00000000 --- a/5-logging/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex -entrypoint: gunicorn -b :$PORT main:app - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/5-logging/bookshelf/__init__.py b/5-logging/bookshelf/__init__.py deleted file mode 100644 index d8f57bb1..00000000 --- a/5-logging/bookshelf/__init__.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # [START setup_logging] - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - # [END setup_logging] - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when debug - # is False - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google+ API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/plus/v1/people/me') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: %s" % resp) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/5-logging/bookshelf/crud.py b/5-logging/bookshelf/crud.py deleted file mode 100644 index f993a432..00000000 --- a/5-logging/bookshelf/crud.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['id'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['displayName'] - data['createdById'] = session['profile']['id'] - - book = get_model().create(data) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/5-logging/bookshelf/model_mongodb.py b/5-logging/bookshelf/model_mongodb.py deleted file mode 100644 index c49d5bbc..00000000 --- a/5-logging/bookshelf/model_mongodb.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bson.objectid import ObjectId -from flask.ext.pymongo import PyMongo - - -builtin_list = list - - -mongo = PyMongo() - - -def _id(id): - if not isinstance(id, ObjectId): - return ObjectId(id) - return id - - -def from_mongo(data): - """ - Translates the MongoDB dictionary format into the format that's expected - by the application. - """ - if not data: - return None - - data['id'] = str(data['_id']) - return data - - -def init_app(app): - mongo.init_app(app) - - -def list(limit=10, cursor=None): - cursor = int(cursor) if cursor else 0 - - results = mongo.db.books.find(skip=cursor, limit=10).sort('title') - books = builtin_list(map(from_mongo, results)) - - next_page = cursor + limit if len(books) == limit else None - return (books, next_page) - - -def list_by_user(user_id, limit=10, cursor=None): - cursor = int(cursor) if cursor else 0 - - results = mongo.db.books\ - .find({'createdById': user_id}, skip=cursor, limit=10)\ - .sort('title') - books = builtin_list(map(from_mongo, results)) - - next_page = cursor + limit if len(books) == limit else None - return (books, next_page) - - -def read(id): - result = mongo.db.books.find_one(_id(id)) - return from_mongo(result) - - -def create(data): - new_id = mongo.db.books.insert(data) - return read(new_id) - - -def update(data, id): - mongo.db.books.update({'_id': _id(id)}, data) - return read(id) - - -def delete(id): - mongo.db.books.remove(_id(id)) diff --git a/5-logging/bookshelf/templates/base.html b/5-logging/bookshelf/templates/base.html deleted file mode 100644 index 4b843b0a..00000000 --- a/5-logging/bookshelf/templates/base.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/5-logging/bookshelf/templates/form.html b/5-logging/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/5-logging/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/5-logging/bookshelf/templates/view.html b/5-logging/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/5-logging/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/5-logging/config.py b/5-logging/config.py deleted file mode 100644 index b64eecd2..00000000 --- a/5-logging/config.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/5-logging/main.py b/5-logging/main.py deleted file mode 100644 index d5697c6c..00000000 --- a/5-logging/main.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/5-logging/requirements-dev.txt b/5-logging/requirements-dev.txt deleted file mode 100644 index cb5c475b..00000000 --- a/5-logging/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -pytest==3.0.5 -pytest-cov==2.4.0 -retrying==1.3.3 -mock==2.0.0 diff --git a/5-logging/requirements.txt b/5-logging/requirements.txt deleted file mode 100644 index 03f56a80..00000000 --- a/5-logging/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 diff --git a/5-logging/tests/conftest.py b/5-logging/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/5-logging/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/5-logging/tests/test_auth.py b/5-logging/tests/test_auth.py deleted file mode 100644 index 7fc7dabc..00000000 --- a/5-logging/tests/test_auth.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'id': 'abc', 'displayName': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/5-logging/tests/test_crud.py b/5-logging/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/5-logging/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/5-logging/tests/test_storage.py b/5-logging/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/5-logging/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/5-logging/tox.ini b/5-logging/tox.ini deleted file mode 100644 index 680033e1..00000000 --- a/5-logging/tox.ini +++ /dev/null @@ -1,23 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report {posargs:tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/6-pubsub/app.yaml b/6-pubsub/app.yaml deleted file mode 100644 index 89284aa0..00000000 --- a/6-pubsub/app.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -runtime: python -env: flex - -# [START entrypoint] -# Instead of using gunicorn directly, we'll use Honcho. Honcho is a python port -# of the Foreman process manager. For the default service, only the -# frontend process is needed. -entrypoint: honcho start -f /app/procfile bookshelf -# [END entrypoint] - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" diff --git a/6-pubsub/bookshelf/__init__.py b/6-pubsub/bookshelf/__init__.py deleted file mode 100644 index 2c6e2112..00000000 --- a/6-pubsub/bookshelf/__init__.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when debug - # is False - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google+ API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/plus/v1/people/me') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: %s" % resp) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/6-pubsub/bookshelf/crud.py b/6-pubsub/bookshelf/crud.py deleted file mode 100644 index b9ecf0f1..00000000 --- a/6-pubsub/bookshelf/crud.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage, tasks -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['id'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['displayName'] - data['createdById'] = session['profile']['id'] - - book = get_model().create(data) - - # [START enqueue] - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - # [END enqueue] - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/6-pubsub/bookshelf/model_cloudsql.py b/6-pubsub/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/6-pubsub/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/6-pubsub/bookshelf/templates/form.html b/6-pubsub/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/6-pubsub/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/6-pubsub/bookshelf/templates/list.html b/6-pubsub/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/6-pubsub/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/6-pubsub/config.py b/6-pubsub/config.py deleted file mode 100644 index d40ef9bc..00000000 --- a/6-pubsub/config.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# Cloud Datastore dataset id, this is the same as your project id. -DATASTORE_DATASET_ID = PROJECT_ID - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/6-pubsub/main.py b/6-pubsub/main.py deleted file mode 100644 index 73d3a303..00000000 --- a/6-pubsub/main.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# [START books_queue] -# Make the queue available at the top-level, this allows you to run -# `psqworker main.books_queue`. We have to use the app's context because -# it contains all the configuration for plugins. -# If you were using another task queue, such as celery or rq, you can use this -# section to configure your queues to work with Flask. -with app.app_context(): - books_queue = bookshelf.tasks.get_books_queue() -# [END books_queue] - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/6-pubsub/monitor.py b/6-pubsub/monitor.py deleted file mode 100644 index 24c4fe63..00000000 --- a/6-pubsub/monitor.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START monitor] -import os -import sys - -from flask import Flask - - -# The app checks this file for the PID of the process to monitor. -PID_FILE = None - - -# Create app to handle health checks and monitor the queue worker. This will -# run alongside the worker, see procfile. -monitor_app = Flask(__name__) - - -# The health check reads the PID file created by psqworker and checks the proc -# filesystem to see if the worker is running. This same pattern can be used for -# rq and celery. -@monitor_app.route('/_ah/health') -def health(): - if not os.path.exists(PID_FILE): - return 'Worker pid not found', 503 - - with open(PID_FILE, 'r') as pidfile: - pid = pidfile.read() - - if not os.path.exists('/proc/{}'.format(pid)): - return 'Worker not running', 503 - - return 'healthy', 200 - - -@monitor_app.route('/') -def index(): - return health() - - -if __name__ == '__main__': - PID_FILE = sys.argv[1] - monitor_app.run('0.0.0.0', 8080) -# [END monitor] diff --git a/6-pubsub/procfile b/6-pubsub/procfile deleted file mode 100644 index 0c4fcec5..00000000 --- a/6-pubsub/procfile +++ /dev/null @@ -1,3 +0,0 @@ -bookshelf: gunicorn -b 0.0.0.0:$PORT main:app -worker: psqworker --pid /tmp/psq.pid main.books_queue -monitor: python monitor.py /tmp/psq.pid diff --git a/6-pubsub/requirements-dev.txt b/6-pubsub/requirements-dev.txt deleted file mode 100644 index 5cf3c0ba..00000000 --- a/6-pubsub/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -mock==2.0.0 -pytest==3.0.5 -pytest-cov==2.4.0 -BeautifulSoup4==4.5.1 -requests==2.12.4 -retrying==1.3.3 diff --git a/6-pubsub/requirements.txt b/6-pubsub/requirements.txt deleted file mode 100644 index dd4195cc..00000000 --- a/6-pubsub/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 -requests[security]==2.12.4 -honcho==0.7.1 -psq==0.5.0 diff --git a/6-pubsub/tests/conftest.py b/6-pubsub/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/6-pubsub/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/6-pubsub/tests/test_crud.py b/6-pubsub/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/6-pubsub/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/6-pubsub/tests/test_end_to_end.py b/6-pubsub/tests/test_end_to_end.py deleted file mode 100644 index 2a64ce8a..00000000 --- a/6-pubsub/tests/test_end_to_end.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -from bs4 import BeautifulSoup -import pytest -import requests -from retrying import retry - - -@pytest.mark.e2e -def test_end_to_end(): - """Tests designed to be run against live environments. - - Unlike the integration tests in the other packages, these tests are - designed to be run against fully-functional live environments. - - To run locally, start both main.py and psq_worker main.books_queue and - run this file. - - It can be run against a live environment by setting the E2E_URL - environment variables before running the tests: - - E2E_URL=http://your-app-id.appspot.com \ - nosetests tests/test_end_to_end.py - """ - - base_url = os.environ.get('E2E_URL', 'http://localhost:8080') - - book_data = { - 'title': 'a confederacy of dunces', - } - - response = requests.post(base_url + '/books/add', data=book_data) - - # There was a 302, so get the book's URL from the redirect. - book_url = response.request.url - book_id = book_url.rsplit('/', 1).pop() - - # Use retry because it will take some indeterminate time for the pub/sub - # message to be processed. - @retry(wait_exponential_multiplier=1000, stop_max_attempt_number=15) - def check_for_updated_data(): - # Check that the book's information was updated. - response = requests.get(book_url) - assert response.status_code == 200 - - soup = BeautifulSoup(response.text, 'html.parser') - - title = soup.find('h4', 'book-title').contents[0].strip() - assert re.search(r'A Confederacy of Dunces', title, re.I) - - author = soup.find('h5', 'book-author').string - assert re.search(r'John Kennedy Toole', author, re.I) - - description = soup.find('p', 'book-description').string - assert re.search(r'Ignatius', description, re.I) - - image_src = soup.find('img', 'book-image')['src'] - image = requests.get(image_src) - assert image.status_code == 200 - - try: - check_for_updated_data() - finally: - # Delete the book we created. - requests.get(base_url + '/books/{}/delete'.format(book_id)) diff --git a/6-pubsub/tests/test_storage.py b/6-pubsub/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/6-pubsub/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/6-pubsub/worker.yaml b/6-pubsub/worker.yaml deleted file mode 100644 index 01ac8a4c..00000000 --- a/6-pubsub/worker.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file specifies your Python application's runtime configuration. -# See https://cloud.google.com/appengine/docs/managed-vms/python/runtime -# for details. - -# [START worker] -service: worker -runtime: python -env: flex - -# Instead of using gunicorn directly, we'll use Honcho. Honcho is a python port -# of the Foreman process manager. For the worker service, both the queue worker -# and the monitor process are needed. -entrypoint: honcho start -f /app/procfile worker monitor - -runtime_config: - python_version: 3 - -beta_settings: - # If using Cloud SQL, uncomment and set this value to the Cloud SQL - # connection name, e.g. - # "project:region:cloudsql-instance" - # You must also update the values in config.py. - # - # cloud_sql_instances: "your-cloudsql-connection-name" -# [END worker] - diff --git a/7-gce/bookshelf/__init__.py b/7-gce/bookshelf/__init__.py deleted file mode 100644 index 01bc5bc1..00000000 --- a/7-gce/bookshelf/__init__.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging - -from flask import current_app, Flask, redirect, request, session, url_for -from google.cloud import error_reporting -import google.cloud.logging -import httplib2 -from oauth2client.contrib.flask_util import UserOAuth2 - - -oauth2 = UserOAuth2() - - -def create_app(config, debug=False, testing=False, config_overrides=None): - app = Flask(__name__) - app.config.from_object(config) - - app.debug = debug - app.testing = testing - - if config_overrides: - app.config.update(config_overrides) - - # Configure logging - if not app.testing: - client = google.cloud.logging.Client(app.config['PROJECT_ID']) - # Attaches a Google Stackdriver logging handler to the root logger - client.setup_logging(logging.INFO) - - # Setup the data model. - with app.app_context(): - model = get_model() - model.init_app(app) - - # Create a health check handler. Health checks are used when running on - # Google Compute Engine by the load balancer to determine which instances - # can serve traffic. Google App Engine also uses health checking, but - # accepts any non-500 response as healthy. - @app.route('/_ah/health') - def health_check(): - return 'ok', 200 - - # Initalize the OAuth2 helper. - oauth2.init_app( - app, - scopes=['email', 'profile'], - authorize_callback=_request_user_info) - - # Add a logout handler. - @app.route('/logout') - def logout(): - # Delete the user's profile and the credentials stored by oauth2. - del session['profile'] - session.modified = True - oauth2.storage.delete() - return redirect(request.referrer or '/') - - # Register the Bookshelf CRUD blueprint. - from .crud import crud - app.register_blueprint(crud, url_prefix='/books') - - # Add a default root route. - @app.route("/") - def index(): - return redirect(url_for('crud.list')) - - # Add an error handler that reports exceptions to Stackdriver Error - # Reporting. Note that this error handler is only used when Debug - # is False - @app.errorhandler(500) - def server_error(e): - client = error_reporting.Client(app.config['PROJECT_ID']) - client.report_exception( - http_context=error_reporting.build_flask_context(request)) - return """ - An internal error occurred. - """, 500 - - return app - - -def get_model(): - model_backend = current_app.config['DATA_BACKEND'] - if model_backend == 'cloudsql': - from . import model_cloudsql - model = model_cloudsql - elif model_backend == 'datastore': - from . import model_datastore - model = model_datastore - elif model_backend == 'mongodb': - from . import model_mongodb - model = model_mongodb - else: - raise ValueError( - "No appropriate databackend configured. " - "Please specify datastore, cloudsql, or mongodb") - - return model - - -def _request_user_info(credentials): - """ - Makes an HTTP request to the Google+ API to retrieve the user's basic - profile information, including full name and photo, and stores it in the - Flask session. - """ - http = httplib2.Http() - credentials.authorize(http) - resp, content = http.request( - 'https://www.googleapis.com/plus/v1/people/me') - - if resp.status != 200: - current_app.logger.error( - "Error while obtaining user profile: %s" % resp) - return None - - session['profile'] = json.loads(content.decode('utf-8')) diff --git a/7-gce/bookshelf/crud.py b/7-gce/bookshelf/crud.py deleted file mode 100644 index 1800aa4f..00000000 --- a/7-gce/bookshelf/crud.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bookshelf import get_model, oauth2, storage, tasks -from flask import Blueprint, current_app, redirect, render_template, request, \ - session, url_for - - -crud = Blueprint('crud', __name__) - - -def upload_image_file(file): - """ - Upload the user-uploaded file to Google Cloud Storage and retrieve its - publicly-accessible URL. - """ - if not file: - return None - - public_url = storage.upload_file( - file.read(), - file.filename, - file.content_type - ) - - current_app.logger.info( - "Uploaded file %s as %s.", file.filename, public_url) - - return public_url - - -@crud.route("/") -def list(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list(cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route("/mine") -@oauth2.required -def list_mine(): - token = request.args.get('page_token', None) - if token: - token = token.encode('utf-8') - - books, next_page_token = get_model().list_by_user( - user_id=session['profile']['id'], - cursor=token) - - return render_template( - "list.html", - books=books, - next_page_token=next_page_token) - - -@crud.route('/') -def view(id): - book = get_model().read(id) - return render_template("view.html", book=book) - - -@crud.route('/add', methods=['GET', 'POST']) -def add(): - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - # If an image was uploaded, update the data to point to the new image. - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - # If the user is logged in, associate their profile with the new book. - if 'profile' in session: - data['createdBy'] = session['profile']['displayName'] - data['createdById'] = session['profile']['id'] - - book = get_model().create(data) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Add", book={}) - - -@crud.route('//edit', methods=['GET', 'POST']) -def edit(id): - book = get_model().read(id) - - if request.method == 'POST': - data = request.form.to_dict(flat=True) - - image_url = upload_image_file(request.files.get('image')) - - if image_url: - data['imageUrl'] = image_url - - book = get_model().update(data, id) - - q = tasks.get_books_queue() - q.enqueue(tasks.process_book, book['id']) - - return redirect(url_for('.view', id=book['id'])) - - return render_template("form.html", action="Edit", book=book) - - -@crud.route('//delete') -def delete(id): - get_model().delete(id) - return redirect(url_for('.list')) diff --git a/7-gce/bookshelf/model_cloudsql.py b/7-gce/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/7-gce/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/7-gce/bookshelf/templates/form.html b/7-gce/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/7-gce/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/7-gce/bookshelf/templates/list.html b/7-gce/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/7-gce/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/7-gce/bookshelf/templates/view.html b/7-gce/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/7-gce/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/7-gce/config.py b/7-gce/config.py deleted file mode 100644 index b64eecd2..00000000 --- a/7-gce/config.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains all of the configuration values for the application. -Update this file with the values for your specific Google Cloud project. -You can create and manage projects at https://console.developers.google.com -""" - -import os - -# The secret key is used by Flask to encrypt session cookies. -SECRET_KEY = 'secret' - -# There are three different ways to store the data in the application. -# You can choose 'datastore', 'cloudsql', or 'mongodb'. Be sure to -# configure the respective settings for the one you choose below. -# You do not have to configure the other data backends. If unsure, choose -# 'datastore' as it does not require any additional configuration. -DATA_BACKEND = 'datastore' - -# Google Cloud Project ID. This can be found on the 'Overview' page at -# https://console.developers.google.com -PROJECT_ID = 'your-project-id' - -# CloudSQL & SQLAlchemy configuration -# Replace the following values the respective values of your Cloud SQL -# instance. -CLOUDSQL_USER = 'root' -CLOUDSQL_PASSWORD = 'your-cloudsql-password' -CLOUDSQL_DATABASE = 'bookshelf' -# Set this value to the Cloud SQL connection name, e.g. -# "project:region:cloudsql-instance". -# You must also update the value in app.yaml. -CLOUDSQL_CONNECTION_NAME = 'your-cloudsql-connection-name' - -# The CloudSQL proxy is used locally to connect to the cloudsql instance. -# To start the proxy, use: -# -# $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 -# -# Port 3306 is the standard MySQL port. If you need to use a different port, -# change the 3306 to a different port number. - -# Alternatively, you could use a local MySQL instance for testing. -LOCAL_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{database}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE) - -# When running on App Engine a unix socket is used to connect to the cloudsql -# instance. -LIVE_SQLALCHEMY_DATABASE_URI = ( - 'mysql+pymysql://{user}:{password}@localhost/{database}' - '?unix_socket=/cloudsql/{connection_name}').format( - user=CLOUDSQL_USER, password=CLOUDSQL_PASSWORD, - database=CLOUDSQL_DATABASE, connection_name=CLOUDSQL_CONNECTION_NAME) - -if os.environ.get('GAE_INSTANCE'): - SQLALCHEMY_DATABASE_URI = LIVE_SQLALCHEMY_DATABASE_URI -else: - SQLALCHEMY_DATABASE_URI = LOCAL_SQLALCHEMY_DATABASE_URI - -# Mongo configuration -# If using mongolab, the connection URI is available from the mongolab control -# panel. If self-hosting on compute engine, replace the values below. -MONGO_URI = 'mongodb://user:password@host:27017/database' - -# Google Cloud Storage and upload settings. -# Typically, you'll name your bucket the same as your project. To create a -# bucket: -# -# $ gsutil mb gs:// -# -# You also need to make sure that the default ACL is set to public-read, -# otherwise users will not be able to see their upload images: -# -# $ gsutil defacl set public-read gs:// -# -# You can adjust the max content length and allow extensions settings to allow -# larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-bucket-name' -MAX_CONTENT_LENGTH = 8 * 1024 * 1024 -ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) - -# OAuth2 configuration. -# This can be generated from the Google Developers Console at -# https://console.developers.google.com/project/_/apiui/credential. -# Note that you will need to add all URLs that your application uses as -# authorized redirect URIs. For example, typically you would add the following: -# -# * http://localhost:8080/oauth2callback -# * https://.appspot.com/oauth2callback. -# -# If you receive a invalid redirect URI error review you settings to ensure -# that the current URI is allowed. -GOOGLE_OAUTH2_CLIENT_ID = \ - 'your-client-id' -GOOGLE_OAUTH2_CLIENT_SECRET = 'your-client-secret' diff --git a/7-gce/gce/deploy.sh b/7-gce/gce/deploy.sh deleted file mode 100755 index 7fc4d374..00000000 --- a/7-gce/gce/deploy.sh +++ /dev/null @@ -1,148 +0,0 @@ -#! /bin/bash -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -ZONE=us-central1-f - -GROUP=frontend-group -TEMPLATE=$GROUP-tmpl -MACHINE_TYPE=f1-micro -IMAGE=debian-8 -STARTUP_SCRIPT=startup-script.sh -SCOPES="userinfo-email,cloud-platform" -TAGS=http-server - -MIN_INSTANCES=1 -MAX_INSTANCES=10 -TARGET_UTILIZATION=0.6 - -SERVICE=frontend-web-service - -# -# Instance group setup -# - -# First we have to create an instance template. -# This template will be used by the instance group -# to create new instances. - -# [START create_template] -gcloud compute instance-templates create $TEMPLATE \ - --image $IMAGE \ - --machine-type $MACHINE_TYPE \ - --scopes $SCOPES \ - --metadata-from-file startup-script=$STARTUP_SCRIPT \ - --tags $TAGS -# [END create_template] - -# Create the managed instance group. - -# [START create_group] -gcloud compute instance-groups managed \ - create $GROUP \ - --base-instance-name $GROUP \ - --size $MIN_INSTANCES \ - --template $TEMPLATE \ - --zone $ZONE -# [END create_group] - -# [START create_named_port] -gcloud compute instance-groups managed set-named-ports \ - $GROUP \ - --named-port http:8080 \ - --zone $ZONE -# [END create_named_port] - -# -# Load Balancer Setup -# - -# A complete HTTP load balancer is structured as follows: -# -# 1) A global forwarding rule directs incoming requests to a target HTTP proxy. -# 2) The target HTTP proxy checks each request against a URL map to determine the -# appropriate backend service for the request. -# 3) The backend service directs each request to an appropriate backend based on -# serving capacity, zone, and instance health of its attached backends. The -# health of each backend instance is verified using either a health check. -# -# We'll create these resources in reverse order: -# service, health check, backend service, url map, proxy. - -# Create a health check -# The load balancer will use this check to keep track of which instances to send traffic to. -# Note that health checks will not cause the load balancer to shutdown any instances. - -# [START create_health_check] -gcloud compute http-health-checks create ah-health-check \ - --request-path /_ah/health \ - --port 8080 -# [END create_health_check] - -# Create a backend service, associate it with the health check and instance group. -# The backend service serves as a target for load balancing. - -# [START create_backend_service] -gcloud compute backend-services create $SERVICE \ - --http-health-check ah-health-check -# [END create_backend_service] - -# [START add_backend_service] -gcloud compute backend-services add-backend $SERVICE \ - --instance-group $GROUP \ - --instance-zone $ZONE -# [END add_backend_service] - -# Create a URL map and web Proxy. The URL map will send all requests to the -# backend service defined above. - -# [START create_url_map] -gcloud compute url-maps create $SERVICE-map \ - --default-service $SERVICE -# [END create_url_map] - -# [START create_http_proxy] -gcloud compute target-http-proxies create $SERVICE-proxy \ - --url-map $SERVICE-map -# [END create_http_proxy] - -# Create a global forwarding rule to send all traffic to our proxy - -# [START create_forwarding_rule] -gcloud compute forwarding-rules create $SERVICE-http-rule \ - --global \ - --target-http-proxy $SERVICE-proxy \ - --port-range 80 -# [END create_forwarding_rule] - -# -# Autoscaler configuration -# -# [START set_autoscaling] -gcloud compute instance-groups managed set-autoscaling \ - $GROUP \ - --max-num-replicas $MAX_INSTANCES \ - --target-load-balancing-utilization $TARGET_UTILIZATION \ - --zone $ZONE -# [END set_autoscaling] - -# [START create_firewall] -gcloud compute firewall-rules create default-allow-http-8080 \ - --allow tcp:8080 \ - --source-ranges 0.0.0.0/0 \ - --target-tags http-server \ - --description "Allow port 8080 access to http-server" -# [END create_firewall] diff --git a/7-gce/gce/deployment_manager/bookshelf.jinja b/7-gce/gce/deployment_manager/bookshelf.jinja deleted file mode 100644 index 8a91dc0f..00000000 --- a/7-gce/gce/deployment_manager/bookshelf.jinja +++ /dev/null @@ -1,168 +0,0 @@ -{# -Copyright 2016 Google Inc. All rights reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -#} - -{# [START all] #} - -{# [START env] #} -{% set NAME = "bookshelf-" + env["deployment"] %} -{% set SERVICE = "bookshelf-" + env["deployment"] + "-frontend" %} -{# [END env] #} - -# -# Instance group setup -# - -# First we have to create an instance template. -# This template will be used by the instance group -# to create new instances. -resources: -- name : {{ NAME }} - type: compute.v1.instanceTemplate - properties: - properties: - tags: - items: - - http-server - disks: - - boot: True - type: PERSISTENT - initializeParams: - sourceImage: {{ properties['machine-image'] }} - diskSizeGb: 10 - diskType: pd-ssd - machineType: {{ properties['machine-type'] }} - serviceAccounts: - - email: default - scopes: {{ properties['scopes'] }} - metadata: - items: - - key: startup-script -{# [START startup] #} - value: | -{{imports['startup-script']|indent(14, true)}} -{# [END startup] #} - networkInterfaces: - - network: global/networks/default - accessConfigs: - - type: ONE_TO_ONE_NAT - name: External NAT - -# Creates the managed instance group. This is responsible for creating -# new instances using the instance template, as well as providing a named -# port the backend service can target -- name: {{ NAME }}-frontend-group - type: compute.v1.instanceGroupManager - properties: - instanceTemplate: $(ref.{{ NAME }}.selfLink) - baseInstanceName: frontend-group - targetSize: 3 - zone: {{ properties['zone'] }} - namedPorts: - - name: http - port: 8080 - - - -# Load Balancer Setup -# - -# A complete HTTP load balancer is structured as follows: -# -# 1) A global forwarding rule directs incoming requests to a target HTTP proxy. -# 2) The target HTTP proxy checks each request against a URL map to determine the -# appropriate backend service for the request. -# 3) The backend service directs each request to an appropriate backend based on -# serving capacity, zone, and instance health of its attached backends. The -# health of each backend instance is verified using either a health check. -# -# We'll create these resources in reverse order: -# service, health check, backend service, url map, proxy. - -# Create a health check -# The load balancer will use this check to keep track of which instances to send traffic to. -# Note that health checks will not cause the load balancer to shutdown any instances. -- name: {{ NAME }}-health-check - type: compute.v1.httpHealthCheck - properties: - requestPath: /_ah/health - port: 8080 - -# Create a backend service, associate it with the health check and instance group. -# The backend service serves as a target for load balancing. -- name: {{ SERVICE }} - type: compute.v1.backendService - properties: - healthChecks: - - $(ref.{{ NAME }}-health-check.selfLink) - portName: http - backends: -{# [START reference] #} - - group: $(ref.{{ NAME }}-frontend-group.instanceGroup) - zone: {{ properties['zone'] }} -{# [END reference] #} - -# Create a URL map and web Proxy. The URL map will send all requests to the -# backend service defined above. -- name: {{ SERVICE }}-map - type: compute.v1.urlMap - properties: - defaultService: $(ref.{{ SERVICE }}.selfLink) - -# This is the actual proxy which uses the URL map to route traffic -# to the backend service -- name: {{ SERVICE }}-proxy - type: compute.v1.targetHttpProxy - properties: - urlMap: $(ref.{{ SERVICE }}-map.selfLink) - -# This is the global forwarding rule which creates an external IP to -# target the http poxy -- name: {{ SERVICE }}-http-rule - type: compute.v1.globalForwardingRule - properties: - target: $(ref.{{ SERVICE }}-proxy.selfLink) - portRange: 80 - -# Creates an autoscaler resource (note that when using the gcloud CLI, -# autoscaling is set as a configuration of the managed instance group -# but autoscaler is a resource so in deployment manager we explicitly -# define it -- name: {{ NAME }}-autoscaler - type: compute.v1.autoscaler - properties: - zone: {{ properties['zone'] }} - target: $(ref.{{ NAME }}-frontend-group.selfLink) - autoscalingPolicy: -{# [START properties] #} - minNumReplicas: {{ properties['min-instances'] }} - maxNumReplicas: {{ properties['max-instances'] }} - loadBalancingUtilization: - utilizationTarget: {{ properties['target-utilization'] }} -{# [END properties] #} - -# Firewall rule that allows traffic to GCE instances with the -# http server tag we created -- name: {{ NAME }}-allow-http - type: compute.v1.firewall - properties: - allowed: - - IPProtocol: tcp - ports: - - 8080 - sourceRanges: - - 0.0.0.0/0 - targetTags: - - http-server - description: "Allow port 8080 access to http-server" - -{# [END all] #} diff --git a/7-gce/gce/deployment_manager/bookshelf.jinja.schema b/7-gce/gce/deployment_manager/bookshelf.jinja.schema deleted file mode 100644 index 526deef6..00000000 --- a/7-gce/gce/deployment_manager/bookshelf.jinja.schema +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START all] - -info: - title: Bookshelf GCE Deploy - author: Google Inc. - description: Creates a GCE Deployment - -imports: -- name: startup-script - path: ../startup-script.sh - -required: -- zone -- machine-type -- min-instances -- max-instances -- scopes - -properties: - zone: - description: Zone to create the resources in. - type: string - machine-type: - description: Type of machine to use - type: string - machine-image: - description: The OS image to use on the machines - type: string - min-instances: - description: The minimum number of VMs the autoscaler will create - type: integer - max-instances: - description: The maximum number of VMs the autoscaler will create - type: integer - target-utilization: - description: The target CPU usage for the autoscaler to base its scaling on - type: number - scopes: - description: A list of scopes to create the VM with - type: array - minItems: 1 - items: - type: string - -# [END all] diff --git a/7-gce/gce/deployment_manager/config.yaml b/7-gce/gce/deployment_manager/config.yaml deleted file mode 100644 index fe343152..00000000 --- a/7-gce/gce/deployment_manager/config.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START all] -imports: -- name: bookshelf.jinja - path: ./bookshelf.jinja - -resources: -- name: bookshelf - type: bookshelf.jinja - properties: - zone: us-central1-f - machine-type: n1-standard-1 - machine-image: https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-8-jessie-v20151104 - min-instances: 1 - max-instances: 10 - target-utilization: 0.6 - scopes: - - https://www.googleapis.com/auth/cloud-platform - -# [END all] diff --git a/7-gce/gce/startup-script.sh b/7-gce/gce/startup-script.sh deleted file mode 100644 index b9b51fc7..00000000 --- a/7-gce/gce/startup-script.sh +++ /dev/null @@ -1,75 +0,0 @@ -#! /bin/bash -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START startup] -set -v - -# Talk to the metadata server to get the project id -PROJECTID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") - -# Install logging monitor. The monitor will automatically pickup logs sent to -# syslog. -# [START logging] -curl -s "https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh" | bash -service google-fluentd restart & -# [END logging] - -# Install dependencies from apt -apt-get update -apt-get install -yq \ - git build-essential supervisor python python-dev python-pip libffi-dev \ - libssl-dev - -# Create a pythonapp user. The application will run as this user. -useradd -m -d /home/pythonapp pythonapp - -# pip from apt is out of date, so make it update itself and install virtualenv. -pip install --upgrade pip virtualenv - -# Get the source code from the Google Cloud Repository -# git requires $HOME and it's not set during the startup script. -export HOME=/root -git config --global credential.helper gcloud.sh -git clone https://source.developers.google.com/p/$PROJECTID /opt/app - -# Install app dependencies -virtualenv /opt/app/7-gce/env -/opt/app/7-gce/env/bin/pip install -r /opt/app/7-gce/requirements.txt - -# Make sure the pythonapp user owns the application code -chown -R pythonapp:pythonapp /opt/app - -# Configure supervisor to start gunicorn inside of our virtualenv and run the -# application. -cat >/etc/supervisor/conf.d/python-app.conf << EOF -[program:pythonapp] -directory=/opt/app/7-gce -command=/opt/app/7-gce/env/bin/gunicorn main:app --bind 0.0.0.0:8080 -autostart=true -autorestart=true -user=pythonapp -# Environment variables ensure that the application runs inside of the -# configured virtualenv. -environment=VIRTUAL_ENV="/opt/app/env/7-gce",PATH="/opt/app/7-gce/env/bin",\ - HOME="/home/pythonapp",USER="pythonapp" -stdout_logfile=syslog -stderr_logfile=syslog -EOF - -supervisorctl reread -supervisorctl update - -# Application should now be running under supervisor -# [END startup] diff --git a/7-gce/gce/teardown.sh b/7-gce/gce/teardown.sh deleted file mode 100755 index 7452db19..00000000 --- a/7-gce/gce/teardown.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#! /bin/bash - -set -x - -ZONE=us-central1-f -gcloud config set compute/zone $ZONE - -GROUP=frontend-group -TEMPLATE=$GROUP-tmpl -SERVICE=frontend-web-service - -gcloud compute instance-groups managed stop-autoscaling $GROUP --zone $ZONE - -gcloud compute forwarding-rules delete $SERVICE-http-rule --global - -gcloud compute target-http-proxies delete $SERVICE-proxy - -gcloud compute url-maps delete $SERVICE-map - -gcloud compute backend-services delete $SERVICE - -gcloud compute http-health-checks delete ah-health-check - -gcloud compute instance-groups managed delete $GROUP - -gcloud compute instance-templates delete $TEMPLATE - -gcloud compute firewall-rules delete default-allow-http-8080 diff --git a/7-gce/main.py b/7-gce/main.py deleted file mode 100644 index 3975fab6..00000000 --- a/7-gce/main.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# Make the queue available at the top-level, this allows you to run -# `psqworker main.books_queue`. We have to use the app's context because -# it contains all the configuration for plugins. -# If you were using another task queue, such as celery or rq, you can use this -# section to configure your queues to work with Flask. -with app.app_context(): - books_queue = bookshelf.tasks.get_books_queue() - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/7-gce/monitor.py b/7-gce/monitor.py deleted file mode 100644 index ecbd098b..00000000 --- a/7-gce/monitor.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from flask import Flask - - -# The app checks this file for the PID of the process to monitor. -PID_FILE = None - - -# Create app to handle health checks and monitor the queue worker. This will -# run alongside the worker, see procfile. -monitor_app = Flask(__name__) - - -# The health check reads the PID file created by psqworker and checks the proc -# filesystem to see if the worker is running. This same pattern can be used for -# rq and celery. -@monitor_app.route('/_ah/health') -def health(): - if not os.path.exists(PID_FILE): - return 'Worker pid not found', 503 - - with open(PID_FILE, 'r') as pidfile: - pid = pidfile.read() - - if not os.path.exists('/proc/{}'.format(pid)): - return 'Worker not running', 503 - - return 'healthy', 200 - - -@monitor_app.route('/') -def index(): - return health() - - -if __name__ == '__main__': - PID_FILE = sys.argv[1] - monitor_app.run('0.0.0.0', 8080) diff --git a/7-gce/procfile b/7-gce/procfile deleted file mode 100644 index 0c4fcec5..00000000 --- a/7-gce/procfile +++ /dev/null @@ -1,3 +0,0 @@ -bookshelf: gunicorn -b 0.0.0.0:$PORT main:app -worker: psqworker --pid /tmp/psq.pid main.books_queue -monitor: python monitor.py /tmp/psq.pid diff --git a/7-gce/requirements-dev.txt b/7-gce/requirements-dev.txt deleted file mode 100644 index 5cf3c0ba..00000000 --- a/7-gce/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -mock==2.0.0 -pytest==3.0.5 -pytest-cov==2.4.0 -BeautifulSoup4==4.5.1 -requests==2.12.4 -retrying==1.3.3 diff --git a/7-gce/requirements.txt b/7-gce/requirements.txt deleted file mode 100644 index dd4195cc..00000000 --- a/7-gce/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 -requests[security]==2.12.4 -honcho==0.7.1 -psq==0.5.0 diff --git a/7-gce/tests/conftest.py b/7-gce/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/7-gce/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/7-gce/tests/test_auth.py b/7-gce/tests/test_auth.py deleted file mode 100644 index 7fc7dabc..00000000 --- a/7-gce/tests/test_auth.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'id': 'abc', 'displayName': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/7-gce/tests/test_crud.py b/7-gce/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/7-gce/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/7-gce/tests/test_storage.py b/7-gce/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/7-gce/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/7-gce/tox.ini b/7-gce/tox.ini deleted file mode 100644 index 430b03fa..00000000 --- a/7-gce/tox.ini +++ /dev/null @@ -1,33 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report -m "not e2e" {posargs: tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST E2E_URL -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:py27-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:py34-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0dfefd93 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The python-samples-owners team is the default owner for anything not +# explicitly taken by someone else. +* @GoogleCloudPlatform/python-samples-reviewers diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6736efd9..c42ef953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,14 +6,12 @@ We'd love to accept your sample apps and patches! Before we can take them, we have to jump a couple of legal hurdles. Please fill out either the individual or corporate Contributor License Agreement -(CLA). +(CLA): * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual CLA] - (https://developers.google.com/open-source/cla/individual). + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA] - (https://developers.google.com/open-source/cla/corporate). + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to @@ -21,15 +19,14 @@ accept your pull requests. ## Contributing A Patch -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owner will respond to your issue promptly. +1. Submit an issue describing your proposed change to the repository in question. +1. The repository owner will respond to your issue promptly. 1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the desired repo, develop and test your code changes. + CLA (see details above). +1. Fork the desired repo, then develop and test your code changes. 1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + you are contributing. Refer to the [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md) and the + [Google Cloud Platform Community Style Guide](https://cloud.google.com/community/tutorials/styleguide) for the recommended coding standards for this organization. 1. Ensure that your code has an appropriate set of unit tests which all pass. 1. Submit a pull request. diff --git a/README.md b/README.md index 4dad11cd..a347e9bb 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,22 @@ # Getting started with Python on Google Cloud Platform -[![Build Status](https://travis-ci.org/GoogleCloudPlatform/getting-started-python.svg)](https://travis-ci.org/GoogleCloudPlatform/getting-started-python) - -This repository is the complete sample code for the [Python Getting Started on Google Cloud Platform](http://cloud.google.com/python) tutorials. Please refer to the tutorials for instructions on configuring, running, and deploying these samples. +This repository is the complete sample code for the [Python Getting Started on Google Cloud Platform](https://cloud.google.com/python/docs/) tutorials. Please refer to the tutorials for instructions on configuring, running, and deploying these samples. The code for the samples is contained in individual folders in this repository. -Note that 7-gce is the final version of the project, including the complete Bookshelf app on Managed VMs in addition to scripts to alternatively deploy on GCE. - - Tutorial | Folder ---------|------- -[Hello world](https://cloud.google.com/python/getting-started/hello-world) | [1-hello-world](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/1-hello-world) -[Structured data](https://cloud.google.com/python/getting-started/using-structured-data) | [2-structured-data](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/2-structured-data) -[Cloud Storage](https://cloud.google.com/python/getting-started/using-cloud-storage) | [3-binary-data](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/3-binary-data) -[Authenticating users](https://cloud.google.com/python/getting-started/authenticate-users) | [4-auth](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/4-auth) -[Logging app events](https://cloud.google.com/python/getting-started/logging-application-events) | [5-logging](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/5-logging) -[Using Cloud Pub/Sub](https://cloud.google.com/python/getting-started/using-pub-sub) | [6-pubsub](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/6-pubsub) -[Deploying to Google Compute Engine](https://cloud.google.com/python/getting-started/run-on-compute-engine) | [7-gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/master/7-gce) +[Getting Started](https://cloud.google.com/python/getting-started/) | [bookshelf](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/bookshelf) +[Background Processing](https://cloud.google.com/python/getting-started/background-processing) | [background](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/background) +[Deploying to Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/docs/quickstarts/deploying-a-language-specific-app) | [in "kubernetes-engine-samples" repo](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/quickstart/python) +[Deploying to Google Compute Engine](https://cloud.google.com/python/tutorials/getting-started-on-compute-engine) | [gce](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/gce) +[Handling Sessions with Firestore](https://cloud.google.com/python/getting-started/session-handling-with-firestore) | [sessions](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/sessions) +[Authenticating Users with IAP](https://cloud.google.com/python/getting-started/authenticate-users) | [authenticating-users](https://github.com/GoogleCloudPlatform/getting-started-python/tree/main/authenticating-users) ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) - ## Licensing * See [LICENSE](LICENSE) diff --git a/2-structured-data/main.py b/authenticating-users/app.yaml similarity index 66% rename from 2-structured-data/main.py rename to authenticating-users/app.yaml index d5697c6c..9010f5b3 100644 --- a/2-structured-data/main.py +++ b/authenticating-users/app.yaml @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config - - -app = bookshelf.create_app(config) - - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) +# [START getting_started_app_yaml] +runtime: python37 +# [END getting_started_app_yaml] diff --git a/authenticating-users/main.py b/authenticating-users/main.py new file mode 100644 index 00000000..9f4b9504 --- /dev/null +++ b/authenticating-users/main.py @@ -0,0 +1,112 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_auth_all] +import sys + +from flask import Flask +app = Flask(__name__) + +CERTS = None +AUDIENCE = None + + +# [START getting_started_auth_certs] +def certs(): + """Returns a dictionary of current Google public key certificates for + validating Google-signed JWTs. Since these change rarely, the result + is cached on first request for faster subsequent responses. + """ + import requests + + global CERTS + if CERTS is None: + response = requests.get( + 'https://www.gstatic.com/iap/verify/public_key' + ) + CERTS = response.json() + return CERTS +# [END getting_started_auth_certs] + + +# [START getting_started_auth_metadata] +def get_metadata(item_name): + """Returns a string with the project metadata value for the item_name. + See https://cloud.google.com/compute/docs/storing-retrieving-metadata for + possible item_name values. + """ + import requests + + endpoint = 'http://metadata.google.internal' + path = '/computeMetadata/v1/project/' + path += item_name + response = requests.get( + '{}{}'.format(endpoint, path), + headers={'Metadata-Flavor': 'Google'} + ) + metadata = response.text + return metadata +# [END getting_started_auth_metadata] + + +# [START getting_started_auth_audience] +def audience(): + """Returns the audience value (the JWT 'aud' property) for the current + running instance. Since this involves a metadata lookup, the result is + cached when first requested for faster future responses. + """ + global AUDIENCE + if AUDIENCE is None: + project_number = get_metadata('numeric-project-id') + project_id = get_metadata('project-id') + AUDIENCE = '/projects/{}/apps/{}'.format( + project_number, project_id + ) + return AUDIENCE +# [END getting_started_auth_audience] + + +# [START getting_started_auth_validate_assertion] +def validate_assertion(assertion): + """Checks that the JWT assertion is valid (properly signed, for the + correct audience) and if so, returns strings for the requesting user's + email and a persistent user ID. If not valid, returns None for each field. + """ + from jose import jwt + + try: + info = jwt.decode( + assertion, + certs(), + algorithms=['ES256'], + audience=audience() + ) + return info['email'], info['sub'] + except Exception as e: + print('Failed to validate assertion: {}'.format(e), file=sys.stderr) + return None, None +# [END getting_started_auth_validate_assertion] + + +# [START getting_started_auth_front_controller] +@app.route('/', methods=['GET']) +def say_hello(): + from flask import request + + assertion = request.headers.get('X-Goog-IAP-JWT-Assertion') + email, id = validate_assertion(assertion) + page = "

Hello {}

".format(email) + return page +# [END getting_started_auth_front_controller] +# [END getting_started_auth_all] diff --git a/authenticating-users/main_test.py b/authenticating-users/main_test.py new file mode 100644 index 00000000..38b934b6 --- /dev/null +++ b/authenticating-users/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def fake_validate(assertion): + if assertion == "Valid": + return "nobody@example.com", "user0001" + else: + return None, None + + +main.validate_assertion = fake_validate + + +def test_home_page(): + client = main.app.test_client() + + # Good request check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Valid"}) + assert "nobody@example.com" in r.text + + # Missing header check + r = client.get("/") + assert "None" in r.text + + # Bad header check + r = client.get("/", headers={"X-Goog-IAP-JWT-Assertion": "Not Valid"}) + assert "None" in r.text diff --git a/authenticating-users/requirements-test.txt b/authenticating-users/requirements-test.txt new file mode 100644 index 00000000..6a3d7bca --- /dev/null +++ b/authenticating-users/requirements-test.txt @@ -0,0 +1 @@ +pytest==7.1.2 \ No newline at end of file diff --git a/authenticating-users/requirements.txt b/authenticating-users/requirements.txt new file mode 100644 index 00000000..0d4a3c36 --- /dev/null +++ b/authenticating-users/requirements.txt @@ -0,0 +1,6 @@ +# [START getting_started_requirements] +Flask==2.2.5 +cryptography==41.0.2 +python-jose[cryptography]==3.3.0 +requests==2.31.0 +# [END getting_started_requirements] diff --git a/background/README.md b/background/README.md new file mode 100644 index 00000000..d79c95fe --- /dev/null +++ b/background/README.md @@ -0,0 +1,17 @@ +Background Processing +--------------------- + +This directory contains an example of doing background processing with App +Engine, Cloud Pub/Sub, Cloud Functions, and Firestore. + +Deploy commands: + +From the app directory: +``` +$ gcloud app deploy +``` + +From the function directory, after creating the PubSub topic: +``` +$ gcloud functions deploy --runtime=python37 --trigger-topic=translate Translate --set-env-vars GOOGLE_CLOUD_PROJECT=my-project +``` diff --git a/background/app/app.yaml b/background/app/app.yaml new file mode 100644 index 00000000..02c0651c --- /dev/null +++ b/background/app/app.yaml @@ -0,0 +1,17 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_background_config] +runtime: python312 +# [END getting_started_background_config] diff --git a/background/app/main.py b/background/app/main.py new file mode 100644 index 00000000..fad93d46 --- /dev/null +++ b/background/app/main.py @@ -0,0 +1,88 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This web app shows translations that have been previously requested, and + provides a form to request a new translation. +""" + +# [START getting_started_background_app_main] +import json +import os + +from flask import Flask, redirect, render_template, request +from google.cloud import firestore, pubsub +from markupsafe import escape + + +app = Flask(__name__) + +# Get client objects to reuse over multiple invocations +db = firestore.Client() +publisher = pubsub.PublisherClient() + +# Keep this list of supported languages up to date +ACCEPTABLE_LANGUAGES = ("de", "en", "es", "fr", "ja", "sw") +# [END getting_started_background_app_main] + + +# [START getting_started_background_app_list] +@app.route("/", methods=["GET"]) +def index(): + """The home page has a list of prior translations and a form to + ask for a new translation. + """ + + doc_list = [] + docs = db.collection("translations").stream() + for doc in docs: + doc_list.append(doc.to_dict()) + + return render_template("index.html", translations=doc_list) + + +# [END getting_started_background_app_list] + + +# [START getting_started_background_app_request] +@app.route("/request-translation", methods=["POST"]) +def translate(): + """Handle a request to translate a string (form field 'v') to a given + language (form field 'lang'), by sending a PubSub message to a topic. + """ + source_string = request.form.get("v", "") + to_language = escape(request.form.get("lang", "")) + + if source_string == "": + return "Invalid request, you must provide a value.", 400 + + if to_language not in ACCEPTABLE_LANGUAGES: + return f"Unsupported language: {to_language}", 400 + + message = { + "Original": source_string, + "Language": to_language, + "Translated": "", + "OriginalLanguage": "", + } + + topic_name = ( + f"projects/{os.getenv('GOOGLE_CLOUD_PROJECT')}/topics/translate" + ) + publisher.publish( + topic=topic_name, data=json.dumps(message).encode("utf-8") + ) + return redirect("/") + + +# [END getting_started_background_app_request] diff --git a/background/app/main_test.py b/background/app/main_test.py new file mode 100644 index 00000000..41b529d3 --- /dev/null +++ b/background/app/main_test.py @@ -0,0 +1,111 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import google.auth +from google.cloud import firestore, pubsub, storage +import main +import pytest + + +credentials, project_id = google.auth.default() +os.environ["GOOGLE_CLOUD_PROJECT"] = project_id +SUBSCRIPTION_NAME = "projects/{}/subscriptions/{}".format( + project_id, "test-" + str(uuid.uuid4()) +) +TOPIC_NAME = "projects/{}/topics/{}".format(project_id, "translate") + + +@pytest.fixture +def db(): + def clear_collection(collection): + """Removes every document from the collection, to make it easy to see + what has been added by the current test run. + """ + for doc in collection.stream(): + doc.reference.delete() + + bucket_name = 'system-test-bucket' + client = firestore.Client() + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + translations = client.collection("translations") + clear_collection(translations) + translations.add( + { + "Original": "A testing message", + "Language": "fr", + "Translated": '"A testing message", but in French', + "OriginalLanguage": "en", + }, + document_id="test translation", + ) + assert bucket in locals() + yield client + + +@pytest.fixture +def publisher(): + client = pubsub.PublisherClient() + yield client + + +@pytest.fixture +def subscriber(): + subscriber = pubsub.SubscriberClient() + subscriber.create_subscription( + request={"name": SUBSCRIPTION_NAME, "topic": TOPIC_NAME} + ) + yield subscriber + subscriber.delete_subscription(request={"subscription": SUBSCRIPTION_NAME}) + + +def test_index(db, publisher): + main.app.testing = True + main.db = db + main.publisher = publisher + client = main.app.test_client() + + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Text to translate" in response_text + assert "but in French" in response_text + + +def test_translate(db, publisher, subscriber): + main.app.testing = True + main.db = db + main.publisher = publisher + client = main.app.test_client() + + r = client.post( + "/request-translation", + data={ + "v": "This is a test", + "lang": "fr", + }, + ) + + assert r.status_code < 400 + + response = subscriber.pull( + request={"subscription": SUBSCRIPTION_NAME, "max_messages": 1}, + timeout=10.0, + ) + assert len(response.received_messages) == 1 + assert b"This is a test" in response.received_messages[0].message.data + assert b"fr" in response.received_messages[0].message.data diff --git a/background/app/requirements.txt b/background/app/requirements.txt new file mode 100644 index 00000000..f70d16b7 --- /dev/null +++ b/background/app/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-firestore==2.18.0 +google-cloud-pubsub==2.23.0 +flask==3.0.3 diff --git a/background/app/templates/index.html b/background/app/templates/index.html new file mode 100644 index 00000000..6f30e24d --- /dev/null +++ b/background/app/templates/index.html @@ -0,0 +1,145 @@ + + + + + + + + + + Translations + + + + + + + + + + + +
+
+
+ + Translate with Background Processing +
+
+
+
+
+
+
+
+
+ + +
+ + +
+
+
+ + + + + + + + + {% for translation in translations %} + + + + + {% endfor %} + +
OriginalTranslation
+ + {{ translation['OriginalLanguage'] }} + + {{ translation['Original'] }} + + + {{ translation['Language'] }} + + {{ translation['Translated'] }} +
+
+ +
+
+
+
+
+ +
+
+
+ + + + diff --git a/background/function/main.py b/background/function/main.py new file mode 100644 index 00000000..ccf94244 --- /dev/null +++ b/background/function/main.py @@ -0,0 +1,106 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This function handles messages posted to a pubsub topic by translating + the data in the message as requested. The message must be a JSON encoded + dictionary with fields: + + Original - the string to translate + Language - the language to translate the string to + + The dictionary may have other fields, which will be ignored. +""" + +# [START getting_started_background_translate_setup] +import base64 +import hashlib +import json + +from google.cloud import firestore +from google.cloud import translate_v2 as translate +# [END getting_started_background_translate_setup] + +# [START getting_started_background_translate_init] +# Get client objects once to reuse over multiple invocations. +xlate = translate.Client() +db = firestore.Client() +# [END getting_started_background_translate_init] + + +# [START getting_started_background_translate_string] +def translate_string(from_string, to_language): + """ Translates a string to a specified language. + + from_string - the original string before translation + + to_language - the language to translate to, as a two-letter code (e.g., + 'en' for english, 'de' for german) + + Returns the translated string and the code for original language + """ + result = xlate.translate(from_string, target_language=to_language) + return result['translatedText'], result['detectedSourceLanguage'] +# [END getting_started_background_translate_string] + + +# [START getting_started_background_translate] +def document_name(message): + """ Messages are saved in a Firestore database with document IDs generated + from the original string and destination language. If the exact same + translation is requested a second time, the result will overwrite the + prior result. + + message - a dictionary with fields named Language and Original, and + optionally other fields with any names + + Returns a unique name that is an allowed Firestore document ID + """ + key = '{}/{}'.format(message['Language'], message['Original']) + hashed = hashlib.sha512(key.encode()).digest() + + # Note that document IDs should not contain the '/' character + name = base64.b64encode(hashed, altchars=b'+-').decode('utf-8') + return name + + +@firestore.transactional +def update_database(transaction, message): + name = document_name(message) + doc_ref = db.collection('translations').document(document_id=name) + + try: + doc_ref.get(transaction=transaction) + except firestore.NotFound: + return # Don't replace an existing translation + + transaction.set(doc_ref, message) + + +def translate_message(event, context): + """ Process a pubsub message requesting a translation + """ + message_data = base64.b64decode(event['data']).decode('utf-8') + message = json.loads(message_data) + + from_string = message['Original'] + to_language = message['Language'] + + to_string, from_language = translate_string(from_string, to_language) + + message['Translated'] = to_string + message['OriginalLanguage'] = from_language + + transaction = db.transaction() + update_database(transaction, message) +# [END getting_started_background_translate] diff --git a/background/function/main_test.py b/background/function/main_test.py new file mode 100644 index 00000000..aa6d3ddb --- /dev/null +++ b/background/function/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +from google.cloud import firestore +import main + + +def clear_collection(collection): + """ Removes every document from the collection, to make it easy to see + what has been added by the current test run. + """ + for doc in collection.stream(): + doc.reference.delete() + + +def test_invocations(): + db = firestore.Client() + main.db = db + + translations = db.collection('translations') + clear_collection(translations) + + event = { + 'data': base64.b64encode(json.dumps({ + 'Original': 'My test message', + 'Language': 'de', + }).encode('utf-8')) + } + + main.translate_message(event, None) + + docs = [doc for doc in translations.stream()] + assert len(docs) == 1 # Should be only the one just created + + message = docs[0].to_dict() + + assert message['Original'] == 'My test message' + assert message['Language'] == 'de' + assert len(message['Translated']) > 0 + assert message['OriginalLanguage'] == 'en' diff --git a/background/function/requirements.txt b/background/function/requirements.txt new file mode 100644 index 00000000..b8e6aaad --- /dev/null +++ b/background/function/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-translate==3.11.1 +google-cloud-firestore==2.11.1 diff --git a/bookshelf/Dockerfile b/bookshelf/Dockerfile new file mode 100644 index 00000000..44c98b38 --- /dev/null +++ b/bookshelf/Dockerfile @@ -0,0 +1,14 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.11-slim + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install --no-cache-dir -r requirements.txt + +# Run the web service on container startup. +ENTRYPOINT [ "gunicorn", "--bind", "0.0.0.0:8080", "main:app" ] \ No newline at end of file diff --git a/bookshelf/app.yaml b/bookshelf/app.yaml new file mode 100644 index 00000000..ac4c378f --- /dev/null +++ b/bookshelf/app.yaml @@ -0,0 +1,2 @@ +runtime: python37 + diff --git a/bookshelf/firestore.py b/bookshelf/firestore.py new file mode 100644 index 00000000..43f18b25 --- /dev/null +++ b/bookshelf/firestore.py @@ -0,0 +1,69 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bookshelf_firestore_client_import] +from google.cloud import firestore +# [END bookshelf_firestore_client_import] + + +def document_to_dict(doc): + if not doc.exists: + return None + doc_dict = doc.to_dict() + doc_dict['id'] = doc.id + return doc_dict + + +def next_page(limit=10, start_after=None): + db = firestore.Client() + + query = db.collection(u'Book').limit(limit).order_by(u'title') + + if start_after: + # Construct a new query starting at this document. + query = query.start_after({u'title': start_after}) + + docs = query.stream() + docs = list(map(document_to_dict, docs)) + + last_title = None + if limit == len(docs): + # Get the last document from the results and set as the last title. + last_title = docs[-1][u'title'] + return docs, last_title + + +def read(book_id): + # [START bookshelf_firestore_client] + db = firestore.Client() + book_ref = db.collection(u'Book').document(book_id) + snapshot = book_ref.get() + # [END bookshelf_firestore_client] + return document_to_dict(snapshot) + + +def update(data, book_id=None): + db = firestore.Client() + book_ref = db.collection(u'Book').document(book_id) + book_ref.set(data) + return document_to_dict(book_ref.get()) + + +create = update + + +def delete(id): + db = firestore.Client() + book_ref = db.collection(u'Book').document(id) + book_ref.delete() diff --git a/bookshelf/images/moby-dick.png b/bookshelf/images/moby-dick.png new file mode 100755 index 00000000..ef789cdb Binary files /dev/null and b/bookshelf/images/moby-dick.png differ diff --git a/bookshelf/main.py b/bookshelf/main.py new file mode 100644 index 00000000..789bec2e --- /dev/null +++ b/bookshelf/main.py @@ -0,0 +1,153 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import firestore +from flask import current_app, flash, Flask, Markup, redirect, render_template +from flask import request, url_for +from google.cloud import error_reporting +import google.cloud.logging +import storage + + +# [START upload_image_file] +def upload_image_file(img): + """ + Upload the user-uploaded file to Google Cloud Storage and retrieve its + publicly-accessible URL. + """ + if not img: + return None + + public_url = storage.upload_file( + img.read(), + img.filename, + img.content_type + ) + + current_app.logger.info( + 'Uploaded file %s as %s.', img.filename, public_url) + + return public_url +# [END upload_image_file] + + +app = Flask(__name__) +app.config.update( + SECRET_KEY='secret', + MAX_CONTENT_LENGTH=8 * 1024 * 1024, + ALLOWED_EXTENSIONS=set(['png', 'jpg', 'jpeg', 'gif']) +) + +app.debug = False +app.testing = False + +# Configure logging +if not app.testing: + logging.basicConfig(level=logging.INFO) + client = google.cloud.logging.Client() + # Attaches a Google Stackdriver logging handler to the root logger + client.setup_logging() + + +@app.route('/') +def list(): + start_after = request.args.get('start_after', None) + books, last_title = firestore.next_page(start_after=start_after) + + return render_template('list.html', books=books, last_title=last_title) + + +@app.route('/books/') +def view(book_id): + book = firestore.read(book_id) + return render_template('view.html', book=book) + + +@app.route('/books/add', methods=['GET', 'POST']) +def add(): + if request.method == 'POST': + data = request.form.to_dict(flat=True) + + # If an image was uploaded, update the data to point to the new image. + image_url = upload_image_file(request.files.get('image')) + + if image_url: + data['imageUrl'] = image_url + + book = firestore.create(data) + + return redirect(url_for('.view', book_id=book['id'])) + + return render_template('form.html', action='Add', book={}) + + +@app.route('/books//edit', methods=['GET', 'POST']) +def edit(book_id): + book = firestore.read(book_id) + + if request.method == 'POST': + data = request.form.to_dict(flat=True) + + # If an image was uploaded, update the data to point to the new image. + image_url = upload_image_file(request.files.get('image')) + + if image_url: + data['imageUrl'] = image_url + + book = firestore.update(data, book_id) + + return redirect(url_for('.view', book_id=book['id'])) + + return render_template('form.html', action='Edit', book=book) + + +@app.route('/books//delete') +def delete(book_id): + firestore.delete(book_id) + return redirect(url_for('.list')) + + +@app.route('/logs') +def logs(): + logging.info('Hey, you triggered a custom log entry. Good job!') + flash(Markup('''You triggered a custom log entry. You can view it in the + Cloud Console''')) + return redirect(url_for('.list')) + + +@app.route('/errors') +def errors(): + raise Exception('This is an intentional exception.') + + +# Add an error handler that reports exceptions to Stackdriver Error +# Reporting. Note that this error handler is only used when debug +# is False +@app.errorhandler(500) +def server_error(e): + client = error_reporting.Client() + client.report_exception( + http_context=error_reporting.build_flask_context(request)) + return """ + An internal error occurred:
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +# This is only used when running locally. When running live, gunicorn runs +# the application. +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/bookshelf/main_test.py b/bookshelf/main_test.py new file mode 100644 index 00000000..249cba17 --- /dev/null +++ b/bookshelf/main_test.py @@ -0,0 +1,169 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import google.auth +import main +import pytest +import requests +from six import BytesIO + + +credentials, project_id = google.auth.default() +os.environ['GOOGLE_CLOUD_PROJECT'] = project_id + + +@pytest.fixture +def app(request): + """This fixture provides a Flask app instance configured for testing. + + It also ensures the tests run within a request context, allowing + any calls to flask.request, flask.current_app, etc. to work.""" + app = main.app + + with app.test_request_context(): + yield app + + +@pytest.fixture +def firestore(): + """This fixture provides a modified version of the app's Firebase model + that tracks all created items and deletes them at the end of the test. + + Any tests that directly or indirectly interact with the database should + use this to ensure that resources are properly cleaned up. + """ + + import firestore + + # Ensure no books exist before running the tests. This typically helps if + # tests somehow left the database in a bad state. + delete_all_books(firestore) + + yield firestore + + # Delete all books that we created during tests. + delete_all_books(firestore) + + +def delete_all_books(firestore): + while True: + books, _ = firestore.next_page(limit=50) + if not books: + break + for book in books: + firestore.delete(book['id']) + + +def test_list(app, firestore): + for i in range(1, 12): + firestore.create({'title': u'Book {0}'.format(i)}) + + with app.test_client() as c: + rv = c.get('/') + + assert rv.status == '200 OK' + + body = rv.data.decode('utf-8') + assert 'Book 1' in body, "Should show books" + assert len(re.findall('

Book', body)) <= 10, ( + "Should not show more than 10 books") + assert 'More' in body, "Should have more than one page" + + +def test_add(app): + data = { + 'title': 'Test Book', + 'author': 'Test Author', + 'publishedDate': 'Test Date Published', + 'description': 'Test Description' + } + + with app.test_client() as c: + rv = c.post('books/add', data=data, follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + assert 'Test Book' in body + assert 'Test Author' in body + assert 'Test Date Published' in body + assert 'Test Description' in body + + +def test_edit(app, firestore): + existing = firestore.create({'title': "Temp Title"}) + + with app.test_client() as c: + rv = c.post( + 'books/%s/edit' % existing['id'], + data={'title': 'Updated Title'}, + follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + assert 'Updated Title' in body + assert 'Temp Title' not in body + + +def test_delete(app, firestore): + existing = firestore.create({'title': "Temp Title"}) + + with app.test_client() as c: + rv = c.get( + 'books/%s/delete' % existing['id'], + follow_redirects=True) + + assert rv.status == '200 OK' + assert not firestore.read(existing['id']) + + +def test_upload_image(app): + data = { + 'title': 'Test Book', + 'author': 'Test Author', + 'publishedDate': 'Test Date Published', + 'description': 'Test Description', + 'image': (BytesIO(b'hello world'), 'hello.jpg') + } + + with app.test_client() as c: + rv = c.post('books/add', data=data, follow_redirects=True) + + assert rv.status == '200 OK' + body = rv.data.decode('utf-8') + + img_tag = re.search(''), + '1337h4x0r.php') + } + + with app.test_client() as c: + rv = c.post('/books/add', data=data, follow_redirects=True) + + # check we weren't pwned + assert rv.status == '400 BAD REQUEST' diff --git a/bookshelf/requirements.txt b/bookshelf/requirements.txt new file mode 100644 index 00000000..27d3d4fe --- /dev/null +++ b/bookshelf/requirements.txt @@ -0,0 +1,7 @@ +Flask==2.2.5 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.9.0 +google-cloud-error-reporting==1.9.1 +google-cloud-logging==3.5.0 +gunicorn==20.1.0 +six==1.16.0 diff --git a/5-logging/bookshelf/storage.py b/bookshelf/storage.py similarity index 73% rename from 5-logging/bookshelf/storage.py rename to bookshelf/storage.py index e030b637..f18b78df 100644 --- a/5-logging/bookshelf/storage.py +++ b/bookshelf/storage.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,30 +15,26 @@ from __future__ import absolute_import import datetime +import os from flask import current_app from google.cloud import storage import six -from werkzeug import secure_filename from werkzeug.exceptions import BadRequest - - -def _get_storage_client(): - return storage.Client( - project=current_app.config['PROJECT_ID']) +from werkzeug.utils import secure_filename def _check_extension(filename, allowed_extensions): - if ('.' not in filename or - filename.split('.').pop().lower() not in allowed_extensions): + file, ext = os.path.splitext(filename) + if (ext.replace('.', '') not in allowed_extensions): raise BadRequest( - "{0} has an invalid name or extension".format(filename)) + '{0} has an invalid name or extension'.format(filename)) def _safe_filename(filename): """ - Generates a safe filename that is unlikely to collide with existing objects - in Google Cloud Storage. + Generates a safe filename that is unlikely to collide with existing + objects in Google Cloud Storage. ``filename.ext`` is transformed into ``filename-YYYY-MM-DD-HHMMSS.ext`` """ @@ -56,15 +52,22 @@ def upload_file(file_stream, filename, content_type): _check_extension(filename, current_app.config['ALLOWED_EXTENSIONS']) filename = _safe_filename(filename) - client = _get_storage_client() - bucket = client.get_bucket(current_app.config['CLOUD_STORAGE_BUCKET']) + bucketname = os.getenv('GOOGLE_STORAGE_BUCKET') or os.getenv( + 'GOOGLE_CLOUD_PROJECT') + '_bucket' + + # [START bookshelf_cloud_storage_client] + client = storage.Client() + bucket = client.bucket(bucketname) blob = bucket.blob(filename) blob.upload_from_string( file_stream, content_type=content_type) + # Ensure the file is publicly readable. + blob.make_public() url = blob.public_url + # [END bookshelf_cloud_storage_client] if isinstance(url, six.binary_type): url = url.decode('utf-8') diff --git a/3-binary-data/bookshelf/templates/base.html b/bookshelf/templates/base.html similarity index 94% rename from 3-binary-data/bookshelf/templates/base.html rename to bookshelf/templates/base.html index a18c7908..0c273faf 100644 --- a/3-binary-data/bookshelf/templates/base.html +++ b/bookshelf/templates/base.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ diff --git a/3-binary-data/bookshelf/templates/form.html b/bookshelf/templates/form.html similarity index 98% rename from 3-binary-data/bookshelf/templates/form.html rename to bookshelf/templates/form.html index 6d5e4d1f..64950d2e 100644 --- a/3-binary-data/bookshelf/templates/form.html +++ b/bookshelf/templates/form.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/5-logging/bookshelf/templates/list.html b/bookshelf/templates/list.html similarity index 80% rename from 5-logging/bookshelf/templates/list.html rename to bookshelf/templates/list.html index 3362f0e2..b255d209 100644 --- a/5-logging/bookshelf/templates/list.html +++ b/bookshelf/templates/list.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,12 @@ {% block content %} +{% with messages = get_flashed_messages() %} + {% for message in messages %} +

{{ message }}

+ {% endfor %} +{% endwith %} +

Books

@@ -31,7 +37,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
@@ -44,10 +50,10 @@

{{book.title}}

No books found

{% endfor %} -{% if next_page_token %} +{% if last_title %}
{% endif %} diff --git a/3-binary-data/bookshelf/templates/view.html b/bookshelf/templates/view.html similarity index 93% rename from 3-binary-data/bookshelf/templates/view.html rename to bookshelf/templates/view.html index 1e509ff0..476db111 100644 --- a/3-binary-data/bookshelf/templates/view.html +++ b/bookshelf/templates/view.html @@ -1,5 +1,5 @@ {# -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
{# [END book_image] #} diff --git a/decrypt-secrets.sh b/decrypt-secrets.sh index 991919ce..7df5428a 100755 --- a/decrypt-secrets.sh +++ b/decrypt-secrets.sh @@ -14,7 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -password=$1 +set -euo pipefail -openssl aes-256-cbc -k "$password" -in secrets.tar.enc -out secrets.tar -d -tar xvf secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +# If there's already a secret file, skip retrieving the secret. +if [[ -f "service-account.json" ]]; then + echo "The secret already exists, skipping." + exit 0 +fi + +gcloud secrets versions access latest \ + --secret="getting-started-python-service-account" \ + --project="${project_id}" \ + > service-account.json diff --git a/encrypt-secrets.sh b/encrypt-secrets.sh index 10f408bb..ddad76db 100755 --- a/encrypt-secrets.sh +++ b/encrypt-secrets.sh @@ -14,11 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -read -s -p "Enter password for encryption: " password -echo +set -euo pipefail -tar cvf secrets.tar {service-account.json,config.py} -openssl aes-256-cbc -k "$password" -in secrets.tar -out secrets.tar.enc -rm secrets.tar +# Always cd to the project root. +readonly root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${root} -travis encrypt "secrets_password=$password" --add +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +readonly project_id="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions add "getting-started-python-service-account" \ + --project="${project_id}" \ + --data-file="service-account.json" diff --git a/gce/README.md b/gce/README.md new file mode 100644 index 00000000..d7c9f1ff --- /dev/null +++ b/gce/README.md @@ -0,0 +1,7 @@ +# Hello World for Python on Google Compute Engine + + This folder contains the sample code for the [Deploying to Google Compute Engine][tutorial-gce] +tutorial. Please refer to the tutorial for instructions on configuring, running, +and deploying this sample. + + [tutorial-gce]: https://cloud.google.com/python/tutorials/getting-started-on-compute-engine diff --git a/gce/add-google-cloud-ops-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh new file mode 100644 index 00000000..63cf2dbd --- /dev/null +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -0,0 +1,519 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *NOTE*: The source of truth for this script is: +# https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +# See https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/installation +# for installation instructions. +# It is committed to this repository to follow security best practices. +# +# +# Add repository for the Google ops agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--uninstall-standalone-logging-agent`: +# Uninstalls the standalone logging agent (`google-fluentd`). +# +# * `--uninstall-standalone-monitoring-agent`: +# Uninstalls the standalone monitoring agent (`stackdriver-agent`). +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-google-cloud-ops-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --dry-run +# +# * To replace standalone agents with the Ops agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --uninstall-standalone-logging-agent --uninstall-standalone-monitoring-agent +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-ops-agent-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + uninstall-standalone-logging-agent) ACTIONS+=('uninstall-standalone-logging-agent') ;; + uninstall-standalone-monitoring-agent) ACTIONS+=('uninstall-standalone-monitoring-agent') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ " ${ACTIONS[*]} " == *\ uninstall\ * || ( " ${ACTIONS[*]} " == *\ remove-repo\ * && " ${ACTIONS[*]} " != *\ also-install\ * )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE + +if [[ " ${ACTIONS[*]} " == *\ also-install*uninstall\ * ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the ops agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/stackdriver/docs/solutions/ops-agent' + +# URL documentation which lists supported platforms for running the ops agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#supported_operating_systems" + +# Packages to install. +AGENT_PACKAGE='google-cloud-ops-agent' +declare -a ADDITIONAL_PACKAGES=() + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-ops-agent ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-ops-agent.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-ops-agent.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-ops-agent.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-ops-agent.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 2.0.1~debian9.13. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}~*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early unless at least one package is installed. + dpkg -l "${packages[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-ops-agent.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-ops-agent/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-ops-agent.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-ops-agent || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-ops-agent.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-ops-agent.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 + } +} + +save_configuration_files() { + local save_dir="/var/lib/google-cloud-ops-agent/saved_configs" + ${DRY_RUN} mkdir -p "${save_dir}" + ${DRY_RUN} cp -rp "$@" "${save_dir}" + echo "$* is now copied over to ${save_dir} folder." +} + +main() { + case "${ID:-}" in + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-logging-agent\ * ]]; then + save_configuration_files "/etc/google-fluentd" + # This will also remove dependent packages, e.g. "google-fluentd-catch-all-config" or "google-fluentd-catch-all-config-structured". + uninstall "google-fluentd" + fi + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-monitoring-agent\ * ]]; then + save_configuration_files "/etc/stackdriver" "/opt/stackdriver/collectd/etc" + uninstall "stackdriver-agent" + fi + if [[ " ${ACTIONS[*]} " == *\ add-repo\ * ]]; then + resolve_version + add_repo + fi + if [[ " ${ACTIONS[*]} " == *\ also-install\ * ]]; then + install_agent + elif [[ " ${ACTIONS[*]} " == *\ uninstall\ * ]]; then + save_configuration_files "/etc/google-cloud-ops-agent" + uninstall "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" + fi + if [[ " ${ACTIONS[*]} " == *\ remove-repo\ * ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/deploy.sh b/gce/deploy.sh new file mode 100644 index 00000000..36ed75ca --- /dev/null +++ b/gce/deploy.sh @@ -0,0 +1,35 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# [START getting_started_gce_create_instance] +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a + +gcloud compute instances create $MY_INSTANCE_NAME \ + --image-family=debian-10 \ + --image-project=debian-cloud \ + --machine-type=g1-small \ + --scopes userinfo-email,cloud-platform \ + --metadata-from-file startup-script=startup-script.sh \ + --zone $ZONE \ + --tags http-server +# [END getting_started_gce_create_instance] + +gcloud compute firewall-rules create default-allow-http-8080 \ + --allow tcp:8080 \ + --source-ranges 0.0.0.0/0 \ + --target-tags http-server \ + --description "Allow port 8080 access to http-server" diff --git a/3-binary-data/main.py b/gce/main.py similarity index 76% rename from 3-binary-data/main.py rename to gce/main.py index d5697c6c..f4435efc 100644 --- a/3-binary-data/main.py +++ b/gce/main.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config +from flask import Flask +app = Flask(__name__) -app = bookshelf.create_app(config) +@app.route('/', methods=['GET']) +def say_hello(): + return "Hello, world!" -# This is only used when running locally. When running live, gunicorn runs -# the application. if __name__ == '__main__': app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/4-auth/main.py b/gce/main_test.py similarity index 67% rename from 4-auth/main.py rename to gce/main_test.py index d5697c6c..6d1f887b 100644 --- a/4-auth/main.py +++ b/gce/main_test.py @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bookshelf -import config +import main -app = bookshelf.create_app(config) +def test_hello(): + main.app.testing = True + client = main.app.test_client() - -# This is only used when running locally. When running live, gunicorn runs -# the application. -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) + r = client.get("/") + assert r.status_code == 200 + response_text = r.data.decode("utf-8") + assert "Hello, world!" in response_text diff --git a/gce/procfile b/gce/procfile new file mode 100644 index 00000000..9d9842b4 --- /dev/null +++ b/gce/procfile @@ -0,0 +1 @@ +hello: /opt/app/gce/env/bin/gunicorn -b 0.0.0.0:8080 main:app diff --git a/gce/python-app.conf b/gce/python-app.conf new file mode 100644 index 00000000..d3a9b17b --- /dev/null +++ b/gce/python-app.conf @@ -0,0 +1,11 @@ +[program:pythonapp] +directory=/opt/app/gce +command=/opt/app/gce/env/bin/honcho start -f ./procfile hello +autostart=true +autorestart=true +user=pythonapp +# Environment variables ensure that the application runs inside of the +# configured virtualenv. +environment=VIRTUAL_ENV="/opt/app/gce/env",PATH="/opt/app/gce/env/bin",HOME="/home/pythonapp",USER="pythonapp" +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/gce/requirements.txt b/gce/requirements.txt new file mode 100644 index 00000000..b655465f --- /dev/null +++ b/gce/requirements.txt @@ -0,0 +1,3 @@ +flask==2.2.5 +honcho==1.1.0 +gunicorn==20.1.0 diff --git a/gce/startup-script.sh b/gce/startup-script.sh new file mode 100644 index 00000000..cc35d30f --- /dev/null +++ b/gce/startup-script.sh @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Echo commands and fail on error +set -ev + +# [START getting_started_gce_startup_script] +# Install or update needed software +apt-get update +apt-get install -yq git supervisor python python-pip python3-distutils +pip install --upgrade pip virtualenv + +# Fetch source code +export HOME=/root +git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app + +# Install Cloud Ops Agent +sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install + +# Account to own server process +useradd -m -d /home/pythonapp pythonapp + +# Python environment setup +virtualenv -p python3 /opt/app/gce/env +/bin/bash -c "source /opt/app/gce/env/bin/activate" +/opt/app/gce/env/bin/pip install -r /opt/app/gce/requirements.txt + +# Set ownership to newly created account +chown -R pythonapp:pythonapp /opt/app + +# Put supervisor configuration in proper place +cp /opt/app/gce/python-app.conf /etc/supervisor/conf.d/python-app.conf + +# Start service via supervisorctl +supervisorctl reread +supervisorctl update +# [END getting_started_gce_startup_script] diff --git a/1-hello-world/main.py b/gce/teardown.sh similarity index 66% rename from 1-hello-world/main.py rename to gce/teardown.sh index 5b398d30..668b1218 100644 --- a/1-hello-world/main.py +++ b/gce/teardown.sh @@ -1,4 +1,6 @@ -# Copyright 2015 Google Inc. + #! /bin/bash + +# Copyright 2019 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START app] -from flask import Flask - - -app = Flask(__name__) - +set -x -@app.route('/') -def hello(): - """Return a friendly HTTP greeting.""" - return 'Hello World!' +MY_INSTANCE_NAME="my-app-instance" +ZONE=us-central1-a +gcloud compute instances delete $MY_INSTANCE_NAME \ + --zone=$ZONE --delete-disks=all -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080) -# [END app] +gcloud compute firewall-rules delete default-allow-http-8080 diff --git a/nox.py b/nox.py deleted file mode 100644 index f2eaf3ca..00000000 --- a/nox.py +++ /dev/null @@ -1,65 +0,0 @@ -from glob import glob -import shutil - -import nox - -REPO_TOOLS_REQ = \ - 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' - -DIRS = [ - # Hello world doesn't have system tests, just a lint test which will be - # covered by the global lint here. - # '1-hello-world', - '2-structured-data', - '3-binary-data', - '4-auth', - '5-logging', - '6-pubsub', - '7-gce'] - - -def session_check_requirements(session): - session.install(REPO_TOOLS_REQ) - - if 'update' in session.posargs: - command = 'update-requirements' - else: - command = 'check-requirements' - - for reqfile in glob('**/requirements*.txt'): - session.run('gcprepotools', command, reqfile) - - -def session_lint(session): - session.install('flake8', 'flake8-import-order') - session.run( - 'flake8', '--exclude=env,.nox,._config.py,.tox', - '--import-order-style=google', '.') - - -def run_test(session, dir, toxargs): - shutil.copy('config.py', dir) - session.chdir(dir) - session.run('tox', *(toxargs or [])) - - -@nox.parametrize('dir', DIRS) -def session_run_tests(session, dir=None, toxargs=None): - """Run all tests for all directories (slow!)""" - run_test(session, dir, toxargs) - - -def session_run_one_test(session): - dir = session.posargs[0] - toxargs = session.posargs[1:] - run_test(session, dir, toxargs) - - -@nox.parametrize('dir', DIRS) -def session_travis(session, dir=None): - """On travis, only run the py3.4 and cloudsql tests.""" - shutil.copy('config.py', dir) - session_run_tests( - session, - dir=dir, - toxargs=['-e', 'py34', '--', '-k', 'cloudsql']) diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..8f5a921c --- /dev/null +++ b/noxfile.py @@ -0,0 +1,62 @@ +from glob import glob +import os + +import nox + +REPO_TOOLS_REQ = \ + 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' + +DIRS = [ + 'authenticating-users', + 'background/app', + 'background/function', + 'gce', + 'sessions', + 'bookshelf', +] + +PYTEST_COMMON_ARGS = ['--junitxml=sponge_log.xml', '-m', 'not e2e'] + + +@nox.session +def check_requirements(session): + session.install(REPO_TOOLS_REQ) + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + for reqfile in glob('**/requirements*.txt'): + session.run('gcp-devrel-py-tools', command, reqfile) + + +@nox.session +def lint(session): + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', '--exclude=env,.nox,._config.py,.tox', + '--import-order-style=google', '.') + + +def run_test(session, dir): + session.install('-r', 'requirements.txt') + session.chdir(dir) + if os.path.exists('requirements.txt'): + session.install('-r', 'requirements.txt') + + session.env['PYTHONPATH'] = os.getcwd() + session.run( + 'pytest', + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # when slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5]) + + +@nox.session(python="3.12") +@nox.parametrize('dir', DIRS) +def run_tests(session, dir=None): + """Run all tests for all directories (slow!)""" + run_test(session, dir) diff --git a/optional-container-engine/bookshelf/model_cloudsql.py b/optional-container-engine/bookshelf/model_cloudsql.py deleted file mode 100644 index d84207d2..00000000 --- a/optional-container-engine/bookshelf/model_cloudsql.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - - -builtin_list = list - - -db = SQLAlchemy() - - -def init_app(app): - # Disable track modifications, as it unnecessarily uses memory. - app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) - db.init_app(app) - - -def from_sql(row): - """Translates a SQLAlchemy model instance into a dictionary""" - data = row.__dict__.copy() - data['id'] = row.id - data.pop('_sa_instance_state') - return data - - -class Book(db.Model): - __tablename__ = 'books' - - id = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(255)) - author = db.Column(db.String(255)) - publishedDate = db.Column(db.String(255)) - imageUrl = db.Column(db.String(255)) - description = db.Column(db.String(4096)) - createdBy = db.Column(db.String(255)) - createdById = db.Column(db.String(255)) - - def __repr__(self): - return " - - - Bookshelf - Python on Google Cloud Platform - - - - - - -
- {% block content %}{% endblock %} -
- {{user}} - - diff --git a/optional-container-engine/bookshelf/templates/form.html b/optional-container-engine/bookshelf/templates/form.html deleted file mode 100644 index 2de32a29..00000000 --- a/optional-container-engine/bookshelf/templates/form.html +++ /dev/null @@ -1,67 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} -

{{action}} book

- -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- - - - - - - - -
- -{% endblock %} diff --git a/optional-container-engine/bookshelf/templates/list.html b/optional-container-engine/bookshelf/templates/list.html deleted file mode 100644 index 3362f0e2..00000000 --- a/optional-container-engine/bookshelf/templates/list.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Books

- - - Add book - - -{% for book in books %} - -{% else %} -

No books found

-{% endfor %} - -{% if next_page_token %} - -{% endif %} - -{% endblock %} diff --git a/optional-container-engine/bookshelf/templates/view.html b/optional-container-engine/bookshelf/templates/view.html deleted file mode 100644 index e654e8ab..00000000 --- a/optional-container-engine/bookshelf/templates/view.html +++ /dev/null @@ -1,53 +0,0 @@ -{# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% extends "base.html" %} - -{% block content %} - -

Book

- - - -
-
- {% if book.imageUrl %} - - {% else %} - - {% endif %} -
-
-

- {{book.title}} - {{book.publishedDate}} -

-
By {{book.author|default('Unknown', True)}}
-

{{book.description}}

- Added by {{book.createdBy|default('Anonymous', True)}} -
-
- -{% endblock %} diff --git a/optional-container-engine/requirements-dev.txt b/optional-container-engine/requirements-dev.txt deleted file mode 100644 index 36d99cea..00000000 --- a/optional-container-engine/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ -tox==2.4.1 -flake8==3.2.1 -flaky==3.3.0 -pytest==3.0.5 -pytest-cov==2.4.0 -BeautifulSoup4==4.5.1 -requests==2.12.4 -retrying==1.3.3 diff --git a/optional-container-engine/requirements.txt b/optional-container-engine/requirements.txt deleted file mode 100644 index e8780a34..00000000 --- a/optional-container-engine/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -Flask==0.11.1 -google-cloud==0.22.0 -gunicorn==19.6.0 -oauth2client==4.0.0 -mock==2.0.0 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.9 -Flask-PyMongo==0.4.1 -PyMongo==3.4.0 -six==1.10.0 -requests[security]==2.12.4 -honcho==0.7.1 -psq==0.5.0 diff --git a/optional-container-engine/tests/conftest.py b/optional-container-engine/tests/conftest.py deleted file mode 100644 index 8123575b..00000000 --- a/optional-container-engine/tests/conftest.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""conftest.py is used to define common test fixtures for pytest.""" - -import bookshelf -import config -from google.cloud.exceptions import ServiceUnavailable -from oauth2client.client import HttpAccessTokenRefreshError -import pytest -from retrying import retry - - -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) -def app(request): - """This fixtures provides a Flask app instance configured for testing. - - Because it's parametric, it will cause every test that uses this fixture - to run three times: one time for each backend (datastore, cloudsql, and - mongodb). - - It also ensures the tests run within a request context, allowing - any calls to flask.request, flask.current_app, etc. to work.""" - app = bookshelf.create_app( - config, - testing=True, - config_overrides={ - 'DATA_BACKEND': request.param - }) - - with app.test_request_context(): - yield app - - -@pytest.yield_fixture -def model(monkeypatch, app): - """This fixture provides a modified version of the app's model that tracks - all created items and deletes them at the end of the test. - - Any tests that directly or indirectly interact with the database should use - this to ensure that resources are properly cleaned up. - - Monkeypatch is provided by pytest and used to patch the model's create - method. - - The app fixture is needed to provide the configuration and context needed - to get the proper model object. - """ - model = bookshelf.get_model() - - # Ensure no books exist before running. This typically helps if tests - # somehow left the database in a bad state. - delete_all_books(model) - - yield model - - # Delete all books that we created during tests. - delete_all_books(model) - - -# The backend data stores can sometimes be flaky. It's useful to retry this -# a few times before giving up. -@retry( - stop_max_attempt_number=3, - wait_exponential_multiplier=100, - wait_exponential_max=2000) -def delete_all_books(model): - while True: - books, _ = model.list(limit=50) - if not books: - break - for book in books: - model.delete(book['id']) - - -def flaky_filter(info, *args): - """Used by flaky to determine when to re-run a test case.""" - _, e, _ = info - return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError)) diff --git a/optional-container-engine/tests/test_auth.py b/optional-container-engine/tests/test_auth.py deleted file mode 100644 index a03e9440..00000000 --- a/optional-container-engine/tests/test_auth.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib - -import bookshelf -from conftest import flaky_filter -from flaky import flaky -import mock -from oauth2client.client import OAuth2Credentials -import pytest - - -@pytest.fixture -def client_with_credentials(app): - """This fixture provides a Flask app test client that has a session - pre-configured with use credentials.""" - credentials = OAuth2Credentials( - 'access_token', - 'client_id', - 'client_secret', - 'refresh_token', - '3600', - None, - 'Test', - id_token={'sub': '123', 'email': 'user@example.com'}, - scopes=('email', 'profile')) - - @contextlib.contextmanager - def inner(): - with app.test_client() as client: - with client.session_transaction() as session: - session['profile'] = {'id': 'abc', 'displayName': 'Test User'} - session['google_oauth2_credentials'] = credentials.to_json() - yield client - - return inner - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestAuth(object): - def test_not_logged_in(self, app): - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Login' in body - - def test_logged_in(self, client_with_credentials): - with client_with_credentials() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test User' in body - - def test_add_anonymous(self, app): - data = { - 'title': 'Test Book', - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Anonymous' in body - - def test_add_logged_in(self, client_with_credentials): - data = { - 'title': 'Test Book', - } - - with client_with_credentials() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Added by Test User' in body - - def test_mine(self, model, client_with_credentials): - # Create two books, one created by the logged in user and one - # created by another user. - model.create({ - 'title': 'Book 1', - 'createdById': 'abc' - }) - - model.create({ - 'title': 'Book 2', - 'createdById': 'def' - }) - - # Check the "My Books" page and make sure only one of the books - # appears. - with client_with_credentials() as c: - rv = c.get('/books/mine') - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Book 1' in body - assert 'Book 2' not in body - - @mock.patch("httplib2.Http") - def test_request_user_info(self, HttpMock): - httpObj = mock.MagicMock() - responseMock = mock.MagicMock(status=200) - httpObj.request = mock.MagicMock( - return_value=(responseMock, b'{"name": "bill"}')) - HttpMock.return_value = httpObj - credentials = mock.MagicMock() - bookshelf._request_user_info(credentials) diff --git a/optional-container-engine/tests/test_crud.py b/optional-container-engine/tests/test_crud.py deleted file mode 100644 index c0d2f40f..00000000 --- a/optional-container-engine/tests/test_crud.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import pytest - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestCrudActions(object): - - def test_list(self, app, model): - for i in range(1, 12): - model.create({'title': u'Book {0}'.format(i)}) - - with app.test_client() as c: - rv = c.get('/books/') - - assert rv.status == '200 OK' - - body = rv.data.decode('utf-8') - assert 'Book 1' in body, "Should show books" - assert len(re.findall('

Book', body)) == 10, ( - "Should not show more than 10 books") - assert 'More' in body, "Should have more than one page" - - def test_add(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description' - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Test Book' in body - assert 'Test Author' in body - assert 'Test Date Published' in body - assert 'Test Description' in body - - def test_edit(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.post( - '/books/%s/edit' % existing['id'], - data={'title': 'Updated Title'}, - follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - assert 'Updated Title' in body - assert 'Temp Title' not in body - - def test_delete(self, app, model): - existing = model.create({'title': "Temp Title"}) - - with app.test_client() as c: - rv = c.get( - '/books/%s/delete' % existing['id'], - follow_redirects=True) - - assert rv.status == '200 OK' - assert not model.read(existing['id']) diff --git a/optional-container-engine/tests/test_end_to_end.py b/optional-container-engine/tests/test_end_to_end.py deleted file mode 100644 index 2a64ce8a..00000000 --- a/optional-container-engine/tests/test_end_to_end.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -from bs4 import BeautifulSoup -import pytest -import requests -from retrying import retry - - -@pytest.mark.e2e -def test_end_to_end(): - """Tests designed to be run against live environments. - - Unlike the integration tests in the other packages, these tests are - designed to be run against fully-functional live environments. - - To run locally, start both main.py and psq_worker main.books_queue and - run this file. - - It can be run against a live environment by setting the E2E_URL - environment variables before running the tests: - - E2E_URL=http://your-app-id.appspot.com \ - nosetests tests/test_end_to_end.py - """ - - base_url = os.environ.get('E2E_URL', 'http://localhost:8080') - - book_data = { - 'title': 'a confederacy of dunces', - } - - response = requests.post(base_url + '/books/add', data=book_data) - - # There was a 302, so get the book's URL from the redirect. - book_url = response.request.url - book_id = book_url.rsplit('/', 1).pop() - - # Use retry because it will take some indeterminate time for the pub/sub - # message to be processed. - @retry(wait_exponential_multiplier=1000, stop_max_attempt_number=15) - def check_for_updated_data(): - # Check that the book's information was updated. - response = requests.get(book_url) - assert response.status_code == 200 - - soup = BeautifulSoup(response.text, 'html.parser') - - title = soup.find('h4', 'book-title').contents[0].strip() - assert re.search(r'A Confederacy of Dunces', title, re.I) - - author = soup.find('h5', 'book-author').string - assert re.search(r'John Kennedy Toole', author, re.I) - - description = soup.find('p', 'book-description').string - assert re.search(r'Ignatius', description, re.I) - - image_src = soup.find('img', 'book-image')['src'] - image = requests.get(image_src) - assert image.status_code == 200 - - try: - check_for_updated_data() - finally: - # Delete the book we created. - requests.get(base_url + '/books/{}/delete'.format(book_id)) diff --git a/optional-container-engine/tests/test_storage.py b/optional-container-engine/tests/test_storage.py deleted file mode 100644 index 531c8827..00000000 --- a/optional-container-engine/tests/test_storage.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from conftest import flaky_filter -from flaky import flaky -import httplib2 -import pytest -from six import BytesIO - - -# Mark all test cases in this class as flaky, so that if errors occur they -# can be retried. This is useful when databases are temporarily unavailable. -@flaky(rerun_filter=flaky_filter) -# Tell pytest to use both the app and model fixtures for all test cases. -# This ensures that configuration is properly applied and that all database -# resources created during tests are cleaned up. These fixtures are defined -# in conftest.py -@pytest.mark.usefixtures('app', 'model') -class TestStorage(object): - - def test_upload_image(self, app): - data = { - 'title': 'Test Book', - 'author': 'Test Author', - 'publishedDate': 'Test Date Published', - 'description': 'Test Description', - 'image': (BytesIO(b'hello world'), 'hello.jpg') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - assert rv.status == '200 OK' - body = rv.data.decode('utf-8') - - img_tag = re.search(''), - '1337h4x0r.php') - } - - with app.test_client() as c: - rv = c.post('/books/add', data=data, follow_redirects=True) - - # check we weren't pwned - assert rv.status == '400 BAD REQUEST' diff --git a/optional-container-engine/tox.ini b/optional-container-engine/tox.ini deleted file mode 100644 index 430b03fa..00000000 --- a/optional-container-engine/tox.ini +++ /dev/null @@ -1,33 +0,0 @@ -[tox] -skipsdist = True -envlist = lint,py27,py34 - -[testenv] -basepython = python2.7 -deps = - -rrequirements.txt - -rrequirements-dev.txt -commands = - py.test --cov=bookshelf --no-success-flaky-report -m "not e2e" {posargs: tests} -passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST E2E_URL -setenv = PYTHONPATH={toxinidir} - -[testenv:py34] -basepython = python3.4 - -[testenv:py27-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:py34-e2e] -basepython = python2.7 -commands = - py.test --no-success-flaky-report -m "e2e" {posargs: tests} - -[testenv:lint] -deps = - flake8 - flake8-import-order -commands = - flake8 --import-order-style=google bookshelf tests diff --git a/optional-container-engine/.dockerignore b/optional-kubernetes-engine/.dockerignore similarity index 77% rename from optional-container-engine/.dockerignore rename to optional-kubernetes-engine/.dockerignore index fdc976d4..c4730648 100644 --- a/optional-container-engine/.dockerignore +++ b/optional-kubernetes-engine/.dockerignore @@ -12,6 +12,9 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log .git +.mypy_cache +.pytest_cache +.hypothesis diff --git a/optional-container-engine/Dockerfile b/optional-kubernetes-engine/Dockerfile similarity index 96% rename from optional-container-engine/Dockerfile rename to optional-kubernetes-engine/Dockerfile index a1f75dc7..69f9d260 100644 --- a/optional-container-engine/Dockerfile +++ b/optional-kubernetes-engine/Dockerfile @@ -2,7 +2,7 @@ # and various os-level packages to allow installation of popular Python # libraries. The source is on github at: # https://github.com/GoogleCloudPlatform/python-docker -FROM gcr.io/google_appengine/python +FROM gcr.io/google-appengine/python # Create a virtualenv for the application dependencies. # If you want to use Python 2, add the -p python2.7 flag. diff --git a/optional-container-engine/Makefile b/optional-kubernetes-engine/Makefile similarity index 78% rename from optional-container-engine/Makefile rename to optional-kubernetes-engine/Makefile index e16a2403..4c1cbf3e 100644 --- a/optional-container-engine/Makefile +++ b/optional-kubernetes-engine/Makefile @@ -6,14 +6,14 @@ all: deploy .PHONY: create-cluster create-cluster: gcloud container clusters create bookshelf \ - --scope "cloud-platform" \ + --scopes "cloud-platform" \ --num-nodes 2 gcloud container clusters get-credentials bookshelf .PHONY: create-bucket create-bucket: - gsutil mb gs://$(GCLOUD_PROJECT) - gsutil defacl set public-read gs://$(GCLOUD_PROJECT) + gcloud storage buckets create gs://$(GCLOUD_PROJECT) + gcloud storage buckets update gs://$(GCLOUD_PROJECT) --predefined-default-object-acl=public-read .PHONY: build build: @@ -21,7 +21,7 @@ build: .PHONY: push push: build - gcloud docker push gcr.io/$(GCLOUD_PROJECT)/bookshelf + gcloud docker -- push gcr.io/$(GCLOUD_PROJECT)/bookshelf .PHONY: template template: @@ -45,6 +45,6 @@ deploy: deploy-frontend deploy-worker create-service .PHONY: delete delete: - -kubectl delete -f bookshelf-frontend.yaml + -kubectl delete -f bookshelf-service.yaml -kubectl delete -f bookshelf-worker.yaml -kubectl delete -f bookshelf-frontend.yaml diff --git a/optional-container-engine/README.md b/optional-kubernetes-engine/README.md similarity index 79% rename from optional-container-engine/README.md rename to optional-kubernetes-engine/README.md index 7b95f925..9acb98b4 100644 --- a/optional-container-engine/README.md +++ b/optional-kubernetes-engine/README.md @@ -1,6 +1,6 @@ -# Deploy Bookshelf to Container Engine/Kubernetes +# Deploy Bookshelf to Google Kubernetes Engine -This optional tutorial will walk you through how to deploy the Bookshelf sample application to [Google Container Engine](https://cloud.google.com/container-engine/). This tutorial is also applicable to [Kubernetes](http://kubernetes.io/) outside of Container Engine, but may require additional steps for external load balancing. +This optional tutorial will walk you through how to deploy the Bookshelf sample application to [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/). This tutorial is also applicable to [Kubernetes](http://kubernetes.io/) outside of Google Kubernetes Engine, but may require additional steps for external load balancing. ## Pre-requisites @@ -12,7 +12,7 @@ This optional tutorial will walk you through how to deploy the Bookshelf sample 4. Install the [Google Cloud SDK](https://cloud.google.com/sdk) - $ curl https://sdk.cloud.google.com | bash + $ curl https://sdk.cloud.google.com | bash $ gcloud init 5. Install [Docker](https://www.docker.com/). @@ -36,8 +36,8 @@ Alternatively, you can use make: The bookshelf application uses [Google Cloud Storage](https://cloud.google.com/storage) to store image files. Create a bucket for your project: - gsutil mb gs:// - gsutil defacl set public-read gs:// + gcloud storage buckets create gs:// + gcloud storage buckets update gs:// --predefined-default-object-acl=public-read Alternatively, you can use make: @@ -49,12 +49,12 @@ Modify config.py and enter your Cloud Project ID into the `PROJECT_ID` and `CLOU ## Build the bookshelf container -Before the application can be deployed to Container Engine, you will need build and push the image to [Google Container Registry](https://cloud.google.com/container-registry/). +Before the application can be deployed to Kubernetes Engine, you will need build and push the image to [Google Container Registry](https://cloud.google.com/container-registry/). - docker build -t gcr.io/your-project-id/bookshelf . - gcloud docker push gcr.io/your-project-id/bookshelf + docker build -t gcr.io//bookshelf . + gcloud docker push gcr.io//bookshelf -Alternativel,y you can use make: +Alternatively, you can use make: make push @@ -92,6 +92,6 @@ Update `bookshelf-worker.yaml` with your Project ID or use `make template`. This Alternatively, you can use make: - make deploy-backend + make deploy-worker Once again, use `kubectl get pods` to check the status of the worker pods. Once the worker pods are up and running, you should be able to create books on the frontend and the workers will handle updating book information in the background. diff --git a/optional-container-engine/bookshelf-frontend.yaml b/optional-kubernetes-engine/bookshelf-frontend.yaml similarity index 92% rename from optional-container-engine/bookshelf-frontend.yaml rename to optional-kubernetes-engine/bookshelf-frontend.yaml index 1a6f56f0..5db2d638 100644 --- a/optional-container-engine/bookshelf-frontend.yaml +++ b/optional-kubernetes-engine/bookshelf-frontend.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf application frontend. The frontend serves # public web traffic. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-frontend @@ -24,9 +24,13 @@ metadata: # The bookshelf frontend replica set ensures that at least 3 # instances of the bookshelf app are running on the cluster. # For more info about Pods see: -# https://cloud.google.com/container-engine/docs/pods/ +# https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 3 + selector: + matchLabels: + app: bookshelf + tier: frontend template: metadata: labels: diff --git a/optional-container-engine/bookshelf-service.yaml b/optional-kubernetes-engine/bookshelf-service.yaml similarity index 83% rename from optional-container-engine/bookshelf-service.yaml rename to optional-kubernetes-engine/bookshelf-service.yaml index d6df8138..db8f1fe7 100644 --- a/optional-container-engine/bookshelf-service.yaml +++ b/optional-kubernetes-engine/bookshelf-service.yaml @@ -13,12 +13,12 @@ # limitations under the License # The bookshelf service provides a load-balancing proxy over the bookshelf -# frontend pods. By specifying the type as a 'LoadBalancer', Container Engine +# frontend pods. By specifying the type as a 'LoadBalancer', Kubernetes Engine # will create an external HTTP load balancer. # For more information about Services see: -# https://cloud.google.com/container-engine/docs/services/ +# https://cloud.google.com/kubernetes-engine/docs/services/ # For more information about external HTTP load balancing see: -# https://cloud.google.com/container-engine/docs/load-balancer +# https://cloud.google.com/kubernetes-engine/docs/load-balancer apiVersion: v1 kind: Service metadata: diff --git a/optional-container-engine/bookshelf-worker.yaml b/optional-kubernetes-engine/bookshelf-worker.yaml similarity index 92% rename from optional-container-engine/bookshelf-worker.yaml rename to optional-kubernetes-engine/bookshelf-worker.yaml index 1c30d650..979437b6 100644 --- a/optional-container-engine/bookshelf-worker.yaml +++ b/optional-kubernetes-engine/bookshelf-worker.yaml @@ -15,7 +15,7 @@ # This file configures the bookshelf task worker. The worker is responsible # for processing book requests and updating book information. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookshelf-worker @@ -24,9 +24,13 @@ metadata: # The bookshelf worker replica set ensures that at least 2 instances of the # bookshelf worker pod are running on the cluster. # For more info about Pods see: -# https://cloud.google.com/container-engine/docs/pods/ +# https://cloud.google.com/kubernetes-engine/docs/pods/ spec: replicas: 2 + selector: + matchLabels: + app: bookshelf + tier: worker template: metadata: labels: diff --git a/optional-container-engine/bookshelf/__init__.py b/optional-kubernetes-engine/bookshelf/__init__.py similarity index 94% rename from optional-container-engine/bookshelf/__init__.py rename to optional-kubernetes-engine/bookshelf/__init__.py index 7f446f33..5fd93b9b 100644 --- a/optional-container-engine/bookshelf/__init__.py +++ b/optional-kubernetes-engine/bookshelf/__init__.py @@ -108,18 +108,18 @@ def get_model(): def _request_user_info(credentials): """ - Makes an HTTP request to the Google+ API to retrieve the user's basic + Makes an HTTP request to the Google OAuth2 API to retrieve the user's basic profile information, including full name and photo, and stores it in the Flask session. """ http = httplib2.Http() credentials.authorize(http) resp, content = http.request( - 'https://www.googleapis.com/plus/v1/people/me') + 'https://www.googleapis.com/oauth2/v3/userinfo') if resp.status != 200: current_app.logger.error( - "Error while obtaining user profile: %s" % resp) + "Error while obtaining user profile: \n%s: %s", resp, content) return None session['profile'] = json.loads(content.decode('utf-8')) diff --git a/optional-container-engine/bookshelf/crud.py b/optional-kubernetes-engine/bookshelf/crud.py similarity index 96% rename from optional-container-engine/bookshelf/crud.py rename to optional-kubernetes-engine/bookshelf/crud.py index 1800aa4f..e07b4053 100644 --- a/optional-container-engine/bookshelf/crud.py +++ b/optional-kubernetes-engine/bookshelf/crud.py @@ -90,8 +90,8 @@ def add(): # If the user is logged in, associate their profile with the new book. if 'profile' in session: - data['createdBy'] = session['profile']['displayName'] - data['createdById'] = session['profile']['id'] + data['createdBy'] = session['profile']['name'] + data['createdById'] = session['profile']['email'] book = get_model().create(data) diff --git a/5-logging/bookshelf/model_cloudsql.py b/optional-kubernetes-engine/bookshelf/model_cloudsql.py similarity index 100% rename from 5-logging/bookshelf/model_cloudsql.py rename to optional-kubernetes-engine/bookshelf/model_cloudsql.py diff --git a/5-logging/bookshelf/model_datastore.py b/optional-kubernetes-engine/bookshelf/model_datastore.py similarity index 100% rename from 5-logging/bookshelf/model_datastore.py rename to optional-kubernetes-engine/bookshelf/model_datastore.py diff --git a/4-auth/bookshelf/model_mongodb.py b/optional-kubernetes-engine/bookshelf/model_mongodb.py similarity index 78% rename from 4-auth/bookshelf/model_mongodb.py rename to optional-kubernetes-engine/bookshelf/model_mongodb.py index 95470b81..ea823cd0 100644 --- a/4-auth/bookshelf/model_mongodb.py +++ b/optional-kubernetes-engine/bookshelf/model_mongodb.py @@ -13,13 +13,13 @@ # limitations under the License. from bson.objectid import ObjectId -from flask.ext.pymongo import PyMongo +from flask_pymongo import PyMongo builtin_list = list -mongo = PyMongo() +mongo = None def _id(id): @@ -28,6 +28,7 @@ def _id(id): return id +# [START from_mongo] def from_mongo(data): """ Translates the MongoDB dictionary format into the format that's expected @@ -38,50 +39,61 @@ def from_mongo(data): data['id'] = str(data['_id']) return data +# [END from_mongo] def init_app(app): - mongo.init_app(app) + global mongo + mongo = PyMongo(app) -def list(limit=10, cursor=None): + +# [START list_by_user] +def list_by_user(user_id, limit=10, cursor=None): cursor = int(cursor) if cursor else 0 - results = mongo.db.books.find(skip=cursor, limit=10).sort('title') + results = mongo.db.books\ + .find({'createdById': user_id}, skip=cursor, limit=10)\ + .sort('title') books = builtin_list(map(from_mongo, results)) next_page = cursor + limit if len(books) == limit else None return (books, next_page) +# [END list_by_user] -# [START list_by_user] -def list_by_user(user_id, limit=10, cursor=None): +# [START list] +def list(limit=10, cursor=None): cursor = int(cursor) if cursor else 0 - results = mongo.db.books\ - .find({'createdById': user_id}, skip=cursor, limit=10)\ - .sort('title') + results = mongo.db.books.find(skip=cursor, limit=10).sort('title') books = builtin_list(map(from_mongo, results)) next_page = cursor + limit if len(books) == limit else None return (books, next_page) -# [END list_by_user] +# [END list] +# [START read] def read(id): - result = mongo.db.books.find_one(_id(id)) + result = mongo.db.books.find_one({'_id': _id(id)}) return from_mongo(result) +# [END read] +# [START create] def create(data): - new_id = mongo.db.books.insert(data) - return read(new_id) + result = mongo.db.books.insert_one(data) + return read(result.inserted_id) +# [END create] +# [START update] def update(data, id): - mongo.db.books.update({'_id': _id(id)}, data) + mongo.db.books.replace_one({'_id': _id(id)}, data) return read(id) +# [END update] def delete(id): - mongo.db.books.remove(_id(id)) + mongo.db.books.delete_one({'_id': _id(id)}) diff --git a/6-pubsub/bookshelf/storage.py b/optional-kubernetes-engine/bookshelf/storage.py similarity index 96% rename from 6-pubsub/bookshelf/storage.py rename to optional-kubernetes-engine/bookshelf/storage.py index e030b637..e8f523a4 100644 --- a/6-pubsub/bookshelf/storage.py +++ b/optional-kubernetes-engine/bookshelf/storage.py @@ -57,7 +57,7 @@ def upload_file(file_stream, filename, content_type): filename = _safe_filename(filename) client = _get_storage_client() - bucket = client.get_bucket(current_app.config['CLOUD_STORAGE_BUCKET']) + bucket = client.bucket(current_app.config['CLOUD_STORAGE_BUCKET']) blob = bucket.blob(filename) blob.upload_from_string( diff --git a/7-gce/bookshelf/tasks.py b/optional-kubernetes-engine/bookshelf/tasks.py similarity index 93% rename from 7-gce/bookshelf/tasks.py rename to optional-kubernetes-engine/bookshelf/tasks.py index e6ca652f..c2fc5a84 100644 --- a/7-gce/bookshelf/tasks.py +++ b/optional-kubernetes-engine/bookshelf/tasks.py @@ -21,16 +21,20 @@ import requests +publisher_client = pubsub.PublisherClient() +subscriber_client = pubsub.SubscriberClient() + + def get_books_queue(): - client = pubsub.Client( - project=current_app.config['PROJECT_ID']) + project = current_app.config['PROJECT_ID'] # Create a queue specifically for processing books and pass in the # Flask application context. This ensures that tasks will have access # to any extensions / configuration specified to the app, such as # models. return psq.Queue( - client, 'books', extra_context=current_app.app_context) + publisher_client, subscriber_client, project, + 'books', extra_context=current_app.app_context) def process_book(book_id): diff --git a/2-structured-data/bookshelf/templates/base.html b/optional-kubernetes-engine/bookshelf/templates/base.html similarity index 100% rename from 2-structured-data/bookshelf/templates/base.html rename to optional-kubernetes-engine/bookshelf/templates/base.html diff --git a/4-auth/bookshelf/templates/form.html b/optional-kubernetes-engine/bookshelf/templates/form.html similarity index 100% rename from 4-auth/bookshelf/templates/form.html rename to optional-kubernetes-engine/bookshelf/templates/form.html diff --git a/4-auth/bookshelf/templates/list.html b/optional-kubernetes-engine/bookshelf/templates/list.html similarity index 95% rename from 4-auth/bookshelf/templates/list.html rename to optional-kubernetes-engine/bookshelf/templates/list.html index 3362f0e2..a80d5b56 100644 --- a/4-auth/bookshelf/templates/list.html +++ b/optional-kubernetes-engine/bookshelf/templates/list.html @@ -31,7 +31,7 @@

Books

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/6-pubsub/bookshelf/templates/view.html b/optional-kubernetes-engine/bookshelf/templates/view.html similarity index 95% rename from 6-pubsub/bookshelf/templates/view.html rename to optional-kubernetes-engine/bookshelf/templates/view.html index e654e8ab..2f337229 100644 --- a/6-pubsub/bookshelf/templates/view.html +++ b/optional-kubernetes-engine/bookshelf/templates/view.html @@ -36,7 +36,7 @@

Book

{% if book.imageUrl %} {% else %} - + {% endif %}
diff --git a/optional-container-engine/config.py b/optional-kubernetes-engine/config.py similarity index 95% rename from optional-container-engine/config.py rename to optional-kubernetes-engine/config.py index f77c8c75..7da9b4cc 100644 --- a/optional-container-engine/config.py +++ b/optional-kubernetes-engine/config.py @@ -78,16 +78,16 @@ # Typically, you'll name your bucket the same as your project. To create a # bucket: # -# $ gsutil mb gs:// +# $ gcloud storage buckets create gs:// # # You also need to make sure that the default ACL is set to public-read, # otherwise users will not be able to see their upload images: # -# $ gsutil defacl set public-read gs:// +# $ gcloud storage buckets update --predefined-default-object-acl=public-read gs:// # # You can adjust the max content length and allow extensions settings to allow # larger or more varied file types if desired. -CLOUD_STORAGE_BUCKET = 'your-project-id' +CLOUD_STORAGE_BUCKET = 'your-bucket-name' MAX_CONTENT_LENGTH = 8 * 1024 * 1024 ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif']) diff --git a/optional-container-engine/main.py b/optional-kubernetes-engine/main.py similarity index 100% rename from optional-container-engine/main.py rename to optional-kubernetes-engine/main.py diff --git a/optional-container-engine/procfile b/optional-kubernetes-engine/procfile similarity index 100% rename from optional-container-engine/procfile rename to optional-kubernetes-engine/procfile diff --git a/optional-kubernetes-engine/requirements-dev.txt b/optional-kubernetes-engine/requirements-dev.txt new file mode 100644 index 00000000..3118b924 --- /dev/null +++ b/optional-kubernetes-engine/requirements-dev.txt @@ -0,0 +1,8 @@ +tox==3.5.3 +flake8==3.6.0 +flaky==3.4.0 +pytest==4.0.1 +pytest-cov==2.6.0 +BeautifulSoup4==4.6.3 +requests==2.20.1 +retrying==1.3.3 diff --git a/optional-kubernetes-engine/requirements.txt b/optional-kubernetes-engine/requirements.txt new file mode 100644 index 00000000..33ba8279 --- /dev/null +++ b/optional-kubernetes-engine/requirements.txt @@ -0,0 +1,16 @@ +Flask==1.0.4 +google-cloud-datastore==1.7.1 +google-cloud-storage==1.23.0 +google-cloud-logging==1.8.0 +google-cloud-error_reporting==0.30.0 +gunicorn==19.9.0 +oauth2client==4.1.3 +mock==2.0.0 +Flask-SQLAlchemy==2.3.2 +PyMySQL==0.9.2 +Flask-PyMongo==2.3.0 +PyMongo==3.7.2 +six==1.11.0 +requests[security]==2.21.0 +honcho==1.0.1 +psq==0.7.0 diff --git a/3-binary-data/tests/conftest.py b/optional-kubernetes-engine/tests/conftest.py similarity index 96% rename from 3-binary-data/tests/conftest.py rename to optional-kubernetes-engine/tests/conftest.py index 8123575b..57a50a1c 100644 --- a/3-binary-data/tests/conftest.py +++ b/optional-kubernetes-engine/tests/conftest.py @@ -22,7 +22,7 @@ from retrying import retry -@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb']) +@pytest.fixture(params=['datastore', 'mongodb']) def app(request): """This fixtures provides a Flask app instance configured for testing. @@ -43,7 +43,7 @@ def app(request): yield app -@pytest.yield_fixture +@pytest.fixture def model(monkeypatch, app): """This fixture provides a modified version of the app's model that tracks all created items and deletes them at the end of the test. diff --git a/6-pubsub/tests/test_auth.py b/optional-kubernetes-engine/tests/test_auth.py similarity index 94% rename from 6-pubsub/tests/test_auth.py rename to optional-kubernetes-engine/tests/test_auth.py index 7fc7dabc..6367608c 100644 --- a/6-pubsub/tests/test_auth.py +++ b/optional-kubernetes-engine/tests/test_auth.py @@ -41,7 +41,10 @@ def client_with_credentials(app): def inner(): with app.test_client() as client: with client.session_transaction() as session: - session['profile'] = {'id': 'abc', 'displayName': 'Test User'} + session['profile'] = { + 'email': 'abc@example.com', + 'name': 'Test User' + } session['google_oauth2_credentials'] = credentials.to_json() yield client @@ -57,7 +60,6 @@ def inner(): # in conftest.py @pytest.mark.usefixtures('app', 'model') class TestAuth(object): - def test_not_logged_in(self, app): with app.test_client() as c: rv = c.get('/books/') @@ -105,12 +107,12 @@ def test_mine(self, model, client_with_credentials): # created by another user. model.create({ 'title': 'Book 1', - 'createdById': 'abc' + 'createdById': 'abc@example.com' }) model.create({ 'title': 'Book 2', - 'createdById': 'def' + 'createdById': 'def@example.com' }) # Check the "My Books" page and make sure only one of the books diff --git a/2-structured-data/tests/test_crud.py b/optional-kubernetes-engine/tests/test_crud.py similarity index 100% rename from 2-structured-data/tests/test_crud.py rename to optional-kubernetes-engine/tests/test_crud.py diff --git a/7-gce/tests/test_end_to_end.py b/optional-kubernetes-engine/tests/test_end_to_end.py similarity index 97% rename from 7-gce/tests/test_end_to_end.py rename to optional-kubernetes-engine/tests/test_end_to_end.py index 2a64ce8a..df80e8a0 100644 --- a/7-gce/tests/test_end_to_end.py +++ b/optional-kubernetes-engine/tests/test_end_to_end.py @@ -52,7 +52,7 @@ def test_end_to_end(): # Use retry because it will take some indeterminate time for the pub/sub # message to be processed. - @retry(wait_exponential_multiplier=1000, stop_max_attempt_number=15) + @retry(wait_exponential_multiplier=5000, stop_max_attempt_number=12) def check_for_updated_data(): # Check that the book's information was updated. response = requests.get(book_url) diff --git a/3-binary-data/tests/test_storage.py b/optional-kubernetes-engine/tests/test_storage.py similarity index 100% rename from 3-binary-data/tests/test_storage.py rename to optional-kubernetes-engine/tests/test_storage.py diff --git a/6-pubsub/tox.ini b/optional-kubernetes-engine/tox.ini similarity index 82% rename from 6-pubsub/tox.ini rename to optional-kubernetes-engine/tox.ini index 430b03fa..8a04f077 100644 --- a/6-pubsub/tox.ini +++ b/optional-kubernetes-engine/tox.ini @@ -1,9 +1,8 @@ [tox] skipsdist = True -envlist = lint,py27,py34 +envlist = lint,py27,py36 [testenv] -basepython = python2.7 deps = -rrequirements.txt -rrequirements-dev.txt @@ -12,16 +11,14 @@ commands = passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST E2E_URL setenv = PYTHONPATH={toxinidir} -[testenv:py34] -basepython = python3.4 [testenv:py27-e2e] basepython = python2.7 commands = py.test --no-success-flaky-report -m "e2e" {posargs: tests} -[testenv:py34-e2e] -basepython = python2.7 +[testenv:py36-e2e] +basepython = python3.6 commands = py.test --no-success-flaky-report -m "e2e" {posargs: tests} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..eb509ac0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = + -v + --tb=native +norecursedirs = .git env lib .tox .nox diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0fd66f54 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +flake8===5.0.4; python_version < '3.8' +flake8==6.0.0; python_version >= '3.8' +pytest==7.3.1 +nox==2023.4.22 +requests==2.31.0 diff --git a/secrets.tar.enc b/secrets.tar.enc index 2942865e..25ccbe61 100644 Binary files a/secrets.tar.enc and b/secrets.tar.enc differ diff --git a/sessions/app.yaml b/sessions/app.yaml new file mode 100644 index 00000000..0b2d0e28 --- /dev/null +++ b/sessions/app.yaml @@ -0,0 +1,17 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_sessions_runtime] +runtime: python37 +# [END getting_started_sessions_runtime] diff --git a/sessions/main.py b/sessions/main.py new file mode 100644 index 00000000..62871ba1 --- /dev/null +++ b/sessions/main.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START getting_started_sessions_all] +import random +from uuid import uuid4 + +from flask import Flask, make_response, request +from google.cloud import firestore + + +app = Flask(__name__) +db = firestore.Client() +sessions = db.collection('sessions') +greetings = [ + 'Hello World', + 'Hallo Welt', + 'Ciao Mondo', + 'Salut le Monde', + 'Hola Mundo', +] + + +@firestore.transactional +def get_session_data(transaction, session_id): + """ Looks up (or creates) the session with the given session_id. + Creates a random session_id if none is provided. Increments + the number of views in this session. Updates are done in a + transaction to make sure no saved increments are overwritten. + """ + if session_id is None: + session_id = str(uuid4()) # Random, unique identifier + + doc_ref = sessions.document(document_id=session_id) + doc = doc_ref.get(transaction=transaction) + if doc.exists: + session = doc.to_dict() + else: + session = { + 'greeting': random.choice(greetings), + 'views': 0 + } + + session['views'] += 1 # This counts as a view + transaction.set(doc_ref, session) + + session['session_id'] = session_id + return session + + +@app.route('/', methods=['GET']) +def home(): + template = '{} views for "{}"' + + transaction = db.transaction() + session = get_session_data(transaction, request.cookies.get('session_id')) + + resp = make_response(template.format( + session['views'], + session['greeting'] + ) + ) + resp.set_cookie('session_id', session['session_id'], httponly=True) + return resp + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080) +# [END getting_started_sessions_all] diff --git a/sessions/main_test.py b/sessions/main_test.py new file mode 100644 index 00000000..06965c08 --- /dev/null +++ b/sessions/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import uuid + +import main +import pytest + + +@pytest.fixture +def client(): + """ Yields a test client, AND creates and later cleans up a + dummy collection for sessions. + """ + main.app.testing = True + + # Override the Firestore collection used for sessions in main + main.sessions = main.db.collection(str(uuid.uuid4())) + + client = main.app.test_client() + yield client + + # Clean up session objects created in test collection + for doc_ref in main.sessions.list_documents(): + doc_ref.delete() + + +def test_session(client): + r = client.get('/') + assert r.status_code == 200 + data = r.data.decode('utf-8') + assert '1 views' in data + + match = re.search('views for "([A-Za-z ]+)"', data) + assert match is not None + greeting = match.group(1) + + r = client.get('/') + assert r.status_code == 200 + data = r.data.decode('utf-8') + assert '2 views' in data + assert greeting in data diff --git a/sessions/requirements-dev.txt b/sessions/requirements-dev.txt new file mode 100644 index 00000000..fe969938 --- /dev/null +++ b/sessions/requirements-dev.txt @@ -0,0 +1 @@ +pytest>=5.0.0 diff --git a/sessions/requirements.txt b/sessions/requirements.txt new file mode 100644 index 00000000..e0e267b5 --- /dev/null +++ b/sessions/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-firestore==2.11.1 +flask==2.2.5