From 9bd47eb80765f7458f12853ee8254e336aa116c8 Mon Sep 17 00:00:00 2001 From: Brittek <83553929+brittek@users.noreply.github.com> Date: Sat, 20 Jan 2024 13:07:30 +1100 Subject: [PATCH 1/2] Initial commit --- .cctemplate | 17 +++ .dockerignore | 28 +++++ .gcloudignore | 29 +++++ .github/CODEOWNERS | 1 + .github/trusted-contribution.yml | 3 + .gitignore | 29 +++++ CONTRIBUTING.md | 28 +++++ Dockerfile | 37 ++++++ LICENSE | 202 +++++++++++++++++++++++++++++++ Procfile | 1 + README.md | 183 ++++++++++++++++++++++++++++ app.py | 57 +++++++++ renovate.json | 21 ++++ requirements-test.txt | 7 ++ requirements.txt | 7 ++ tasks.py | 148 ++++++++++++++++++++++ test/__init__.py | 0 test/advance.cloudbuild.yaml | 110 +++++++++++++++++ test/common.sh | 56 +++++++++ test/conftest.py | 29 +++++ test/test_app.py | 26 ++++ test/test_system.py | 32 +++++ utils/logging.py | 89 ++++++++++++++ utils/metadata.py | 46 +++++++ 24 files changed, 1186 insertions(+) create mode 100644 .cctemplate create mode 100644 .dockerignore create mode 100644 .gcloudignore create mode 100644 .github/CODEOWNERS create mode 100644 .github/trusted-contribution.yml create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Procfile create mode 100644 README.md create mode 100644 app.py create mode 100644 renovate.json create mode 100644 requirements-test.txt create mode 100644 requirements.txt create mode 100644 tasks.py create mode 100644 test/__init__.py create mode 100644 test/advance.cloudbuild.yaml create mode 100644 test/common.sh create mode 100644 test/conftest.py create mode 100644 test/test_app.py create mode 100644 test/test_system.py create mode 100644 utils/logging.py create mode 100644 utils/metadata.py diff --git a/.cctemplate b/.cctemplate new file mode 100644 index 000000000000..10adc1ca3dc2 --- /dev/null +++ b/.cctemplate @@ -0,0 +1,17 @@ +{ + "name": "cloud-run-microservice-template-python", + "metadata": { + "version": "0.1.0" + }, + "templates": [ + { + "path": "./", + "name": "Cloud Run Microservice Template - Python", + "description": "Basic microservice template for Cloud Run", + "languages": [ + "Python", + "Dockerfile" + ] + } + ] +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..527924e58315 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The .dockerignore file excludes files from the container build process +# when building with the Docker CLI. +# +# https://docs.docker.com/engine/reference/builder/#dockerignore-file + +# Exclude locally installed dependencies +venv/ + +# Exclude "build-time" ignore files. +.dockerignore +.gcloudignore + +# Exclude git history and configuration. +.gitignore \ No newline at end of file diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000000..03c46d882598 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The .gcloudignore file excludes file from upload to Cloud Build. +# If this file is deleted, gcloud will default to .gitignore. +# +# https://cloud.google.com/cloud-build/docs/speeding-up-builds#gcloudignore +# https://cloud.google.com/sdk/gcloud/reference/topic/gcloudignore + +# Ignore everything specified in .gitignore +#!include:.gitignore + +# Exclude locally installed dependencies +venv/ + +# Exclude git history and configuration. +.git/ +.gitignore \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..9f6730a1880e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @GoogleCloudPlatform/torus-dpe diff --git a/.github/trusted-contribution.yml b/.github/trusted-contribution.yml new file mode 100644 index 000000000000..31a45ab20a80 --- /dev/null +++ b/.github/trusted-contribution.yml @@ -0,0 +1,3 @@ +annotations: + - type: comment + text: "/gcbrun" diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..a482b74f00f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A gitignore file specifies intentionally untracked files that Git should ignore. + +# Exclude locally installed dependencies +venv/ +__pycache__/ + +# Exclude local IDE settings +.idea/ +.vscode/ + +# Exclude testing cache +.pytest_cache/ + +# macOS +**/.DS_Store \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000000..e296e9968ce2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to become a contributor and submit your own code +## Contributor License Agreements +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. +Please fill out either the individual or corporate Contributor License Agreement +(CLA). + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA] + (https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA] + (https://developers.google.com/open-source/cla/corporate). +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll be able to +accept your pull requests. +## Contributing A Patch +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the sample to which + you are contributing. Refer to the + [Google Cloud Platform Samples Style Guide] + (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + recommended coding standards for this organization. +1. Ensure that your code has an appropriate set of unit tests which all pass. +1. Submit a pull request. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..03e119e7372f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2021 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use the official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.10.6-slim + +# Allow statements and log messages to immediately appear in the Cloud Run logs +ENV PYTHONUNBUFFERED 1 + +# Create and change to the app directory. +WORKDIR /usr/src/app + +# Copy application dependency manifests to the container image. +# Copying this separately prevents re-running pip install on every code change. +COPY requirements.txt ./ + +# Install dependencies. +RUN pip install -r requirements.txt + +# Copy local code to the container image. +COPY . ./ + +# Run the web service on container startup. +# Use gunicorn webserver with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 app:app diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Procfile b/Procfile new file mode 100644 index 000000000000..19fb0c65978f --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn --bind :8080 --workers 1 --threads 8 --timeout 0 app:app \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000000..c2fb6a0bd5da --- /dev/null +++ b/README.md @@ -0,0 +1,183 @@ +# Cloud Run Template Microservice + +A template repository for a Cloud Run microservice, written in Python + +[![Run on Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run) + +## Prerequisite + +* Enable the Cloud Run API via the [console](https://console.cloud.google.com/apis/library/run.googleapis.com?_ga=2.124941642.1555267850.1615248624-203055525.1615245957) or CLI: + +```bash +gcloud services enable run.googleapis.com +``` + +## Features + +* **Flask**: Web server framework +* **Buildpack support** Tooling to build production-ready container images from source code and without a Dockerfile +* **Dockerfile**: Container build instructions, if needed to replace buildpack for custom build +* **SIGTERM handler**: Catch termination signal for cleanup before Cloud Run stops the container +* **Service metadata**: Access service metadata, project ID and region, at runtime +* **Local development utilities**: Auto-restart with changes and prettify logs +* **Structured logging w/ Log Correlation** JSON formatted logger, parsable by Cloud Logging, with [automatic correlation of container logs to a request log](https://cloud.google.com/run/docs/logging#correlate-logs). +* **Unit and System tests**: Basic unit and system tests setup for the microservice +* **Task definition and execution**: Uses [invoke](http://www.pyinvoke.org/) to execute defined tasks in `tasks.py`. + +## Local Development + +### Cloud Code + +This template works with [Cloud Code](https://cloud.google.com/code), an IDE extension +to let you rapidly iterate, debug, and run code on Kubernetes and Cloud Run. + +Learn how to use Cloud Code for: + +* Local development - [VSCode](https://cloud.google.com/code/docs/vscode/developing-a-cloud-run-service), [IntelliJ](https://cloud.google.com/code/docs/intellij/developing-a-cloud-run-service) + +* Local debugging - [VSCode](https://cloud.google.com/code/docs/vscode/debugging-a-cloud-run-service), [IntelliJ](https://cloud.google.com/code/docs/intellij/debugging-a-cloud-run-service) + +* Deploying a Cloud Run service - [VSCode](https://cloud.google.com/code/docs/vscode/deploying-a-cloud-run-service), [IntelliJ](https://cloud.google.com/code/docs/intellij/deploying-a-cloud-run-service) +* Creating a new application from a custom template (`.template/templates.json` allows for use as an app template) - [VSCode](https://cloud.google.com/code/docs/vscode/create-app-from-custom-template), [IntelliJ](https://cloud.google.com/code/docs/intellij/create-app-from-custom-template) + +### CLI tooling + +To run the `invoke` commands below, install [`invoke`](https://www.pyinvoke.org/index.html) system wide: + +```bash +pip install invoke +``` + +Invoke will handle establishing local virtual environments, etc. Task definitions can be found in `tasks.py`. + +#### Local development + +1. Set Project Id: + ```bash + export GOOGLE_CLOUD_PROJECT= + ``` +2. Start the server with hot reload: + ```bash + invoke dev + ``` + +#### Deploying a Cloud Run service + +1. Set Project Id: + ```bash + export GOOGLE_CLOUD_PROJECT= + ``` + +1. Enable the Artifact Registry API: + ```bash + gcloud services enable artifactregistry.googleapis.com + ``` + +1. Create an Artifact Registry repo: + ```bash + export REPOSITORY="samples" + export REGION=us-central1 + gcloud artifacts repositories create $REPOSITORY --location $REGION --repository-format "docker" + ``` + +1. Use the gcloud credential helper to authorize Docker to push to your Artifact Registry: + ```bash + gcloud auth configure-docker + ``` + +2. Build the container using a buildpack: + ```bash + invoke build + ``` +3. Deploy to Cloud Run: + ```bash + invoke deploy + ``` + +### Run sample tests + +1. [Pass credentials via `GOOGLE_APPLICATION_CREDENTIALS` env var](https://cloud.google.com/docs/authentication/production#passing_variable): + ```bash + export GOOGLE_APPLICATION_CREDENTIALS="[PATH]" + ``` + +2. Set Project Id: + ```bash + export GOOGLE_CLOUD_PROJECT= + ``` +3. Run unit tests + ```bash + invoke test + ``` + +4. Run system tests + ```bash + gcloud builds submit \ + --config test/advance.cloudbuild.yaml \ + --substitutions 'COMMIT_SHA=manual,REPO_NAME=manual' + ``` + The Cloud Build configuration file will build and deploy the containerized service + to Cloud Run, run tests managed by pytest, then clean up testing resources. This configuration restricts public + access to the test service. Therefore, service accounts need to have the permission to issue ID tokens for request authorization: + * Enable Cloud Run, Cloud Build, Artifact Registry, and IAM APIs: + ```bash + gcloud services enable run.googleapis.com cloudbuild.googleapis.com iamcredentials.googleapis.com artifactregistry.googleapis.com + ``` + + * Set environment variables. + ```bash + export PROJECT_ID="$(gcloud config get-value project)" + export PROJECT_NUMBER="$(gcloud projects describe $(gcloud config get-value project) --format='value(projectNumber)')" + ``` + + * Create an Artifact Registry repo (or use another already created repo): + ```bash + export REPOSITORY="samples" + export REGION=us-central1 + gcloud artifacts repositories create $REPOSITORY --location $REGION --repository-format "docker" + ``` + + * Create service account `token-creator` with `Service Account Token Creator` and `Cloud Run Invoker` roles. + ```bash + gcloud iam service-accounts create token-creator + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:token-creator@$PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/iam.serviceAccountTokenCreator" + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:token-creator@$PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/run.invoker" + ``` + + * Add `Service Account Token Creator` role to the Cloud Build service account. + ```bash + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ + --role="roles/iam.serviceAccountTokenCreator" + ``` + + * Cloud Build also requires permission to deploy Cloud Run services and administer artifacts: + + ```bash + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ + --role="roles/run.admin" + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ + --role="roles/iam.serviceAccountUser" + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ + --role="roles/artifactregistry.repoAdmin" + ``` + +## Maintenance & Support + +This repo performs basic periodic testing for maintenance. Please use the issue tracker for bug reports, features requests and submitting pull requests. + +## Contributions + +Please see the [contributing guidelines](CONTRIBUTING.md) + +## License + +This library is licensed under Apache 2.0. Full license text is available in [LICENSE](LICENSE). diff --git a/app.py b/app.py new file mode 100644 index 000000000000..01013b6661fd --- /dev/null +++ b/app.py @@ -0,0 +1,57 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import sys +from types import FrameType + +from flask import Flask + +from utils.logging import logger + +app = Flask(__name__) + + +@app.route("/") +def hello() -> str: + # Use basic logging with custom fields + logger.info(logField="custom-entry", arbitraryField="custom-entry") + + # https://cloud.google.com/run/docs/logging#correlate-logs + logger.info("Child logger with trace Id.") + + return "Hello, World!" + + +def shutdown_handler(signal_int: int, frame: FrameType) -> None: + logger.info(f"Caught Signal {signal.strsignal(signal_int)}") + + from utils.logging import flush + + flush() + + # Safely exit program + sys.exit(0) + + +if __name__ == "__main__": + # Running application locally, outside of a Google Cloud Environment + + # handles Ctrl-C termination + signal.signal(signal.SIGINT, shutdown_handler) + + app.run(host="localhost", port=8080, debug=True) +else: + # handles Cloud Run container termination + signal.signal(signal.SIGTERM, shutdown_handler) diff --git a/renovate.json b/renovate.json new file mode 100644 index 000000000000..4938d22053aa --- /dev/null +++ b/renovate.json @@ -0,0 +1,21 @@ +{ + "extends": [ + "config:base" + ], + "prConcurrentLimit": 0, + "rebaseWhen": "never", + "masterIssue": true, + "pip_requirements": { + "fileMatch": ["requirements-test.txt"] + }, + "packageRules": [ + { + "matchPackageNames": ["pytest"], + "matchUpdateTypes": ["minor", "major"] + } + ], + "constraints": { + "python": "3.8.6" + }, + "schedule": ["before 8am on the first day of the month"] +} diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000000..b0742e8e86f6 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,7 @@ +pip==23.3.2 +invoke==2.0.0 +pytest==7.1.3 +black==23.7.0 +flake8==6.0.0 +flake8-annotations==3.0.0 +flake8-import-order==0.18.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..95d0fcce5801 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask==3.0.0 +gunicorn==21.2.0 + +requests==2.31.0 +structlog==22.1.0 + +google-auth==2.3.2 diff --git a/tasks.py b/tasks.py new file mode 100644 index 000000000000..030963e0fdcf --- /dev/null +++ b/tasks.py @@ -0,0 +1,148 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Invoke tasks - a Python-equivelant of a Makefile or Rakefile. +# http://www.pyinvoke.org/ + +# LINTING NOTE: invoke doesn't support annotations in task signatures. +# https://github.com/pyinvoke/invoke/issues/777 +# Workaround: add " # noqa: ANN001, ANN201" + +import os +import sys +from typing import List + +from invoke import task + +venv = "source ./venv/bin/activate" +GOOGLE_CLOUD_PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT") +REGION = os.environ.get("REGION", "us-central1") + + +@task +def require_project(c): # noqa: ANN001, ANN201 + """(Check) Require GOOGLE_CLOUD_PROJECT be defined""" + if GOOGLE_CLOUD_PROJECT is None: + print("GOOGLE_CLOUD_PROJECT not defined. Required for task") + sys.exit(1) + + +@task +def require_venv(c, test_requirements=False, quiet=True): # noqa: ANN001, ANN201 + """(Check) Require that virtualenv is setup, requirements installed""" + + c.run("python -m venv venv") + quiet_param = " -q" if quiet else "" + + with c.prefix(venv): + c.run(f"pip install -r requirements.txt {quiet_param}") + + if test_requirements: + c.run(f"pip install -r requirements-test.txt {quiet_param}") + + +@task +def require_venv_test(c): # noqa: ANN001, ANN201 + """(Check) Require that virtualenv is setup, requirements (incl. test) installed""" + require_venv(c, test_requirements=True) + + +@task +def setup_virtualenv(c): # noqa: ANN001, ANN201 + """Create virtualenv, and install requirements, with output""" + require_venv(c, test_requirements=True, quiet=False) + + +@task(pre=[require_venv]) +def start(c): # noqa: ANN001, ANN201 + """Start the web service""" + with c.prefix(venv): + c.run("python app.py") + + +@task(pre=[require_venv]) +def dev(c): # noqa: ANN001, ANN201 + """Start the web service in a development environment, with fast reload""" + with c.prefix(venv): + c.run("FLASK_ENV=development python app.py") + + +@task(pre=[require_venv]) +def lint(c): # noqa: ANN001, ANN201 + """Run linting checks""" + with c.prefix(venv): + local_names = _determine_local_import_names(".") + c.run( + "flake8 --exclude venv " + "--max-line-length=88 " + "--import-order-style=google " + f"--application-import-names {','.join(local_names)} " + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202" + ) + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +@task(pre=[require_venv]) +def fix(c): # noqa: ANN001, ANN201 + """Apply linting fixes""" + with c.prefix(venv): + c.run("black *.py **/*.py --force-exclude venv") + c.run("isort --profile google *.py **/*.py") + + +@task(pre=[require_project]) +def build(c): # noqa: ANN001, ANN201 + """Build the service into a container image""" + c.run( + f"gcloud builds submit --pack " + f"image={REGION}-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual" + ) + + +@task(pre=[require_project]) +def deploy(c): # noqa: ANN001, ANN201 + """Deploy the container into Cloud Run (fully managed)""" + c.run( + "gcloud run deploy microservice-template " + f"--image {REGION}-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual " + f"--platform managed --region {REGION}" + ) + + +@task(pre=[require_venv_test]) +def test(c): # noqa: ANN001, ANN201 + """Run unit tests""" + with c.prefix(venv): + c.run("pytest test/test_app.py") + + +@task(pre=[require_venv_test]) +def system_test(c): # noqa: ANN001, ANN201 + """Run system tests""" + with c.prefix(venv): + c.run("pytest test/test_system.py") diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/advance.cloudbuild.yaml b/test/advance.cloudbuild.yaml new file mode 100644 index 000000000000..6a8259e68225 --- /dev/null +++ b/test/advance.cloudbuild.yaml @@ -0,0 +1,110 @@ +# Copyright 2021 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: "Install Invoke and Pip" + name: $_PYTHON_VERSION + entrypoint: python + args: ["-m", "pip", "install", "invoke", "-U", "pip", "--user"] + + - id: "Setup virtualenv" + name: $_PYTHON_VERSION + entrypoint: python + args: ["-m", "invoke", "setup-virtualenv"] + + - id: "Lint" + name: $_PYTHON_VERSION + entrypoint: python + args: ["-m", "invoke", "lint"] + + - id: "Run Unit Tests" + name: $_PYTHON_VERSION + entrypoint: python + args: ["-m", "invoke", "test"] + + # Setup resources for system tests + - id: "Build Container Image" + name: "gcr.io/k8s-skaffold/pack" + entrypoint: pack + args: + - build + - "$_GCR_HOSTNAME/$PROJECT_ID/$_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA" # Tag docker image with git commit SHA + - "--builder=gcr.io/buildpacks/builder:latest" + - "--path=." + + - id: "Push Container Image" + name: "gcr.io/cloud-builders/docker:latest" + entrypoint: /bin/bash + timeout: 60s + args: + - "-c" + - | + while ! docker push "$_GCR_HOSTNAME/$PROJECT_ID/$_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA"; do + sleep 2 + done + + - id: "Deploy to Cloud Run" + name: "gcr.io/cloud-builders/gcloud:latest" + entrypoint: /bin/bash + args: + - "-c" + - | + gcloud run deploy ${_SERVICE_NAME}-$BUILD_ID \ + --image $_GCR_HOSTNAME/$PROJECT_ID/$_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA \ + --region ${_DEPLOY_REGION} \ + --no-allow-unauthenticated + + - id: "Retrieve Service URL" + name: "gcr.io/cloud-builders/gcloud:latest" + entrypoint: /bin/bash + args: + - "-c" + - | + source /workspace/test/common.sh + echo $(get_url ${BUILD_ID}) > _service_url + echo $(get_idtoken) > _id_token + env: + - "_SERVICE_NAME=${_SERVICE_NAME}" + - "_DEPLOY_REGION=${_DEPLOY_REGION}" + - "PROJECT_ID=${PROJECT_ID}" + + - id: "Run System Tests" + name: "${_PYTHON_VERSION}" + entrypoint: /bin/bash + args: + - "-c" + - | + export BASE_URL=$(cat _service_url) + export ID_TOKEN=$(cat _id_token) + python -m invoke system-test + + # Clean up system test resources + + - id: "Delete image and service" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + gcloud artifacts docker images delete $_GCR_HOSTNAME/$PROJECT_ID/$_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA --quiet + gcloud run services delete ${_SERVICE_NAME}-$BUILD_ID \ + --region ${_DEPLOY_REGION} --quiet + +options: + dynamic_substitutions: true + substitution_option: "ALLOW_LOOSE" + +substitutions: + _GCR_HOSTNAME: us-central1-docker.pkg.dev + _SERVICE_NAME: microservice-template + _DEPLOY_REGION: us-central1 + _PYTHON_VERSION: python:3.8.6-slim # matches buildpack latest + _REPOSITORY: samples diff --git a/test/common.sh b/test/common.sh new file mode 100644 index 000000000000..756e0d4f9109 --- /dev/null +++ b/test/common.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +# common.sh +# Provides utility functions commonly needed across Cloud Build pipelines. +# +# This is expected to be used from cloud-run-template.cloudbuild.yaml and +# should be "forked" into an individual sample that does not provide the same +# environment variables and workspace. +# +# It is kept separate for two reasons: +# 1. Simplicity of cloudbuild.yaml files. +# 2. Easier evaluation of security implications in changes to get_idtoken(). +# +# Usage +# If you do not need to fork this script, directly source it in your YAML file: +# +# ``` +# . /testing/cloudbuild-templates/common.sh +# echo $(get_url) > _service_url +# ``` +## + +# Retrieve Cloud Run service URL - +# Cloud Run URLs are not deterministic. +get_url() { + bid=$(test "$1" && echo "$1" || cat _short_id) + gcloud run services describe ${_SERVICE_NAME}-${bid} \ + --format 'value(status.url)' \ + --region ${_DEPLOY_REGION} \ + --platform managed +} + +# Retrieve Id token to make an aunthenticated request - +# Impersonate service account, token-creator@, since +# Cloud Build does not natively mint identity tokens. +get_idtoken() { + curl -X POST -H "content-type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -d "{\"audience\": \"$(cat _service_url)\"}" \ + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/token-creator@${PROJECT_ID}.iam.gserviceaccount.com:generateIdToken" | \ + python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" +} diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000000..4f35c9f2bae7 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask +from flask.testing import FlaskClient +import pytest + +from app import app as flask_app + + +@pytest.fixture +def app() -> None: + yield flask_app + + +@pytest.fixture +def client(app: flask.app.Flask) -> FlaskClient: + return app.test_client() diff --git a/test/test_app.py b/test/test_app.py new file mode 100644 index 000000000000..89a921967eca --- /dev/null +++ b/test/test_app.py @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask +from flask.testing import FlaskClient + + +def test_get_index(app: flask.app.Flask, client: FlaskClient) -> None: + res = client.get("/") + assert res.status_code == 200 + + +def test_post_index(app: flask.app.Flask, client: FlaskClient) -> None: + res = client.post("/") + assert res.status_code == 405 diff --git a/test/test_system.py b/test/test_system.py new file mode 100644 index 000000000000..b59125d5e1f2 --- /dev/null +++ b/test/test_system.py @@ -0,0 +1,32 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import flask +from flask.testing import FlaskClient +import requests + + +def test_system(app: flask.app.Flask, client: FlaskClient) -> None: + + BASE_URL = os.environ.get("BASE_URL") + assert BASE_URL, "Cloud Run service URL not found" + + ID_TOKEN = os.environ.get("ID_TOKEN") + assert ID_TOKEN, "Unable to acquire an ID token" + + resp = requests.get(BASE_URL, headers={"Authorization": f"Bearer {ID_TOKEN}"}) + assert resp.status_code == 200 + assert resp.text == "Hello, World!" diff --git a/utils/logging.py b/utils/logging.py new file mode 100644 index 000000000000..c1babf0bcbff --- /dev/null +++ b/utils/logging.py @@ -0,0 +1,89 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +from flask import request +import structlog + +from utils import metadata + + +def field_name_modifier( + logger: structlog._loggers.PrintLogger, log_method: str, event_dict: Dict +) -> Dict: + """Changes the keys for some of the fields, + to match Cloud Logging's expectations + https://cloud.google.com/run/docs/logging#special-fields + """ + # structlog example adapted from + # https://github.com/ymotongpoo/cloud-logging-configurations/blob/master/python/structlog/main.py + + event_dict["severity"] = event_dict["level"] + del event_dict["level"] + + if "event" in event_dict: + event_dict["message"] = event_dict["event"] + del event_dict["event"] + return event_dict + + +def trace_modifier( + logger: structlog._loggers.PrintLogger, log_method: str, event_dict: Dict +) -> Dict: + """Adds Tracing correlation + https://cloud.google.com/run/docs/logging#correlate-logs + """ + # Only attempt to get the context if in a request + if request: + + trace_header = request.headers.get("X-Cloud-Trace-Context") + # Only append the trace if it exists in the request + if trace_header: + trace = trace_header.split("/") + project = metadata.get_project_id() + event_dict[ + "logging.googleapis.com/trace" + ] = f"projects/{project}/traces/{trace[0]}" + return event_dict + + +def getJSONLogger() -> structlog._config.BoundLoggerLazyProxy: + """Create a JSON logger using the field name and trace modifiers created above""" + # extend using https://www.structlog.org/en/stable/processors.html + structlog.configure( + processors=[ + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + field_name_modifier, + trace_modifier, + structlog.processors.TimeStamper("iso"), + structlog.processors.JSONRenderer(), + ], + wrapper_class=structlog.stdlib.BoundLogger, + ) + return structlog.get_logger() + + +logger = getJSONLogger() + + +def flush() -> None: + # Setting PYTHONUNBUFFERED in Dockerfile/Buildpack ensured no buffering + + # https://docs.python.org/3/library/logging.html#logging.shutdown + # When the logging module is imported, it registers this + # function as an exit handler (see atexit), so normally + # there’s no need to do that manually. + pass diff --git a/utils/metadata.py b/utils/metadata.py new file mode 100644 index 000000000000..9d07501e4eaf --- /dev/null +++ b/utils/metadata.py @@ -0,0 +1,46 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import requests + +METADATA_URI = "http://metadata.google.internal/computeMetadata/v1/" + + +def get_project_id() -> str: + """Use the 'google-auth-library' to make a request to the metadata server or + default to Application Default Credentials in your local environment.""" + _, project = google.auth.default() + return project + + +def get_service_region() -> str: + """Get region from local metadata server + Region in format: projects/PROJECT_NUMBER/regions/REGION""" + slug = "instance/region" + data = requests.get(METADATA_URI + slug, headers={"Metadata-Flavor": "Google"}) + return data.content + + +def authenticated_request(url: str, method: str) -> str: + """Make a request with an ID token to a protected service + https://cloud.google.com/functions/docs/securing/authenticating#functions-bearer-token-example-python""" + + auth_req = google.auth.transport.requests.Request() + id_token = google.oauth2.id_token.fetch_id_token(auth_req, url) + + resp = requests.request( + method, url, headers={"Authorization": f"Bearer {id_token}"} + ) + return resp.content From e0b6b3f600d505c98493322499bf36600310e5bb Mon Sep 17 00:00:00 2001 From: Sourcery AI Date: Fri, 9 Feb 2024 16:03:49 +0000 Subject: [PATCH 2/2] 'Refactored by Sourcery' --- utils/logging.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/logging.py b/utils/logging.py index c1babf0bcbff..c3fa3da68f7a 100644 --- a/utils/logging.py +++ b/utils/logging.py @@ -48,9 +48,7 @@ def trace_modifier( # Only attempt to get the context if in a request if request: - trace_header = request.headers.get("X-Cloud-Trace-Context") - # Only append the trace if it exists in the request - if trace_header: + if trace_header := request.headers.get("X-Cloud-Trace-Context"): trace = trace_header.split("/") project = metadata.get_project_id() event_dict[