diff --git a/.coveragerc b/.coveragerc index 6a18f761d2c4..8aff2b6744ca 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,7 +6,8 @@ omit = */_generated/*.py # Packages in the "google.cloud" package that we don't own. */google/cloud/gapic/* -fail_under = 100 + */google/cloud/grpc/* + */google/cloud/proto/* show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..fcf34e569555 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,52 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +# Bu Sun Kim is the default owner for anything not explicitly +# taken by someone else. +* @busunkim96 + +################################## +# Storage and Database (SoDa) team +################################## +bigtable/ @tseaver @crwilcox @frankyn +docs/bigtable/ @tseaver @crwilcox @frankyn + +datastore/ @tseaver @crwilcox @frankyn +docs/datastore @tseaver @crwilcox @frankyn + +storage/ @tseaver @crwilcox @frankyn +docs/storage @tseaver @crwilcox @frankyn + +firestore/ @tseaver @crwilcox @frankyn +docs/firestore/ @tseaver @crwilcox @frankyn + +spanner/ @tseaver @crwilcox @frankyn +docs/spanner @tseaver @crwilcox @frankyn + +############################### +# Analytics and ML (AniML) team +############################### + +# BigQuery suite of APIs +bigquery/ @googleapis/api-bigquery +docs/bigquery/ @googleapis/api-bigquery + +bigquery_datatransfer/ @googleapis/api-bigquery +docs/bigquery_datatransfer/ @googleapis/api-bigquery + +bigquery_storage/ @googleapis/api-bigquery +docs/bigquery_storage/ @googleapis/api-bigquery + +# Pubsub +pubsub/ @anguillanneuf @plamut +docs/pubsub @anguillanneuf @plamut + + +# Tim Swast is the primary author of Runtime Config. +runtimeconfig/ @tswast +docs/runtimeconfig/ @tswast + + +# Doug Greiman and Angela Li are the owners of trace. +trace/ @liyanhui1228 @duggelz +docs/trace/ @liyanhui1228 @duggelz diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000000..939e5341e74d --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000000..1ca956493631 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,43 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +Please run down the following list and make sure you've tried the usual "quick fixes": + + - Search the issues already opened: https://github.com/googleapis/google-cloud-python/issues + - Check for answers on StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-python + +If you are still having issues, please be sure to include as much information as possible: + +#### Environment details + +1. Specify the API at the beginning of the title (for example, "BigQuery: ...") + General, Core, and Other are also allowed as types +2. OS type and version +3. Python version and virtual environment information: `python --version` +4. google-cloud- version: `pip show google-` or `pip freeze` + +#### Steps to reproduce + + 1. ? + +#### Code example + +```python +# example +``` + +#### Stack trace +``` +# example +``` + +Making sure to follow these steps will guarantee the quickest resolution possible. + +Thanks! diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000000..6365857f33c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this library + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + + **Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + **Describe the solution you'd like** +A clear and concise description of what you want to happen. + **Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + **Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md new file mode 100644 index 000000000000..995869032125 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support_request.md @@ -0,0 +1,7 @@ +--- +name: Support request +about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. + +--- + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/.gitignore b/.gitignore index e2eb2ef7a2e5..87a4d1890975 100644 --- a/.gitignore +++ b/.gitignore @@ -25,8 +25,11 @@ pip-log.txt # Unit test / coverage reports .coverage +.nox .tox .cache +.pytest_cache +htmlcov # Translations *.mo @@ -42,8 +45,14 @@ pip-log.txt # JetBrains .idea +# VS Code +.vscode + # Built documentation docs/_build +*/docs/_build +docs/_build_doc2dash +bigquery/docs/generated # Virtual environment env/ @@ -53,10 +62,12 @@ coverage.xml system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. -scripts/pylintrc_reduced +pylintrc +pylintrc.test # Directories used for creating generated PB2 files generated_python/ cloud-bigtable-client/ googleapis-pb/ grpc_python_venv/ +pytype_output/ diff --git a/.kokoro/build.sh b/.kokoro/build.sh new file mode 100755 index 000000000000..39634b7d6d52 --- /dev/null +++ b/.kokoro/build.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd github/google-cloud-python + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Setup firestore account credentials +export FIRESTORE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/firebase-credentials.json + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json + +# Setup project id. +export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") + +# Find out if this package was modified. +# Temporarily use Thea's fork of ci-diff-helper w/ Kokoro support. +python3.6 -m pip install --quiet git+https://github.com/theacodes/ci-diff-helper.git +python3.6 test_utils/scripts/get_target_packages_kokoro.py > ~/target_packages +cat ~/target_packages + +if [[ ! -n $(grep -x "$PACKAGE" ~/target_packages) ]]; then + echo "$PACKAGE was not modified, returning." + exit; +fi + +cd "$PACKAGE" + +# Remove old nox +python3.6 -m pip uninstall --yes --quiet nox-automation + +# Install nox +python3.6 -m pip install --upgrade --quiet nox +python3.6 -m nox --version + +python3.6 -m nox diff --git a/.kokoro/continuous/api_core.cfg b/.kokoro/continuous/api_core.cfg new file mode 100644 index 000000000000..00ae35f46e2f --- /dev/null +++ b/.kokoro/continuous/api_core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "api_core" +} diff --git a/.kokoro/continuous/asset.cfg b/.kokoro/continuous/asset.cfg new file mode 100644 index 000000000000..61c3fa9c5b4b --- /dev/null +++ b/.kokoro/continuous/asset.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "asset" +} diff --git a/.kokoro/continuous/automl.cfg b/.kokoro/continuous/automl.cfg new file mode 100644 index 000000000000..5081664abb56 --- /dev/null +++ b/.kokoro/continuous/automl.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "automl" +} diff --git a/.kokoro/continuous/bigquery.cfg b/.kokoro/continuous/bigquery.cfg new file mode 100644 index 000000000000..69eabb20803c --- /dev/null +++ b/.kokoro/continuous/bigquery.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery" +} diff --git a/.kokoro/continuous/bigquery_datatransfer.cfg b/.kokoro/continuous/bigquery_datatransfer.cfg new file mode 100644 index 000000000000..47e510d21a44 --- /dev/null +++ b/.kokoro/continuous/bigquery_datatransfer.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_datatransfer" +} diff --git a/.kokoro/continuous/bigquery_storage.cfg b/.kokoro/continuous/bigquery_storage.cfg new file mode 100644 index 000000000000..7f72bfed0587 --- /dev/null +++ b/.kokoro/continuous/bigquery_storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_storage" +} diff --git a/.kokoro/continuous/bigtable.cfg b/.kokoro/continuous/bigtable.cfg new file mode 100644 index 000000000000..de67c4844de9 --- /dev/null +++ b/.kokoro/continuous/bigtable.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigtable" +} diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg new file mode 100644 index 000000000000..5dec06191476 --- /dev/null +++ b/.kokoro/continuous/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "google-cloud-python/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-python/.kokoro/build.sh" +} diff --git a/.kokoro/continuous/container.cfg b/.kokoro/continuous/container.cfg new file mode 100644 index 000000000000..8d06b5285bf2 --- /dev/null +++ b/.kokoro/continuous/container.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "container" +} diff --git a/.kokoro/continuous/containeranalysis.cfg b/.kokoro/continuous/containeranalysis.cfg new file mode 100644 index 000000000000..ae80ed8d8bc6 --- /dev/null +++ b/.kokoro/continuous/containeranalysis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "containeranalysis" +} diff --git a/.kokoro/continuous/core.cfg b/.kokoro/continuous/core.cfg new file mode 100644 index 000000000000..68b0d504429c --- /dev/null +++ b/.kokoro/continuous/core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "core" +} diff --git a/.kokoro/continuous/datacatalog.cfg b/.kokoro/continuous/datacatalog.cfg new file mode 100644 index 000000000000..fc0371e5eb11 --- /dev/null +++ b/.kokoro/continuous/datacatalog.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datacatalog" +} diff --git a/.kokoro/continuous/datalabeling.cfg b/.kokoro/continuous/datalabeling.cfg new file mode 100644 index 000000000000..a1c53b81884b --- /dev/null +++ b/.kokoro/continuous/datalabeling.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datalabeling" +} diff --git a/.kokoro/continuous/dataproc.cfg b/.kokoro/continuous/dataproc.cfg new file mode 100644 index 000000000000..d9ea2e670aea --- /dev/null +++ b/.kokoro/continuous/dataproc.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dataproc" +} diff --git a/.kokoro/continuous/datastore.cfg b/.kokoro/continuous/datastore.cfg new file mode 100644 index 000000000000..7cf6a9c0dc8f --- /dev/null +++ b/.kokoro/continuous/datastore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datastore" +} diff --git a/.kokoro/continuous/dlp.cfg b/.kokoro/continuous/dlp.cfg new file mode 100644 index 000000000000..931126d3e719 --- /dev/null +++ b/.kokoro/continuous/dlp.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dlp" +} diff --git a/.kokoro/continuous/dns.cfg b/.kokoro/continuous/dns.cfg new file mode 100644 index 000000000000..845e3e148060 --- /dev/null +++ b/.kokoro/continuous/dns.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dns" +} diff --git a/.kokoro/continuous/error_reporting.cfg b/.kokoro/continuous/error_reporting.cfg new file mode 100644 index 000000000000..99a86c522b9a --- /dev/null +++ b/.kokoro/continuous/error_reporting.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "error_reporting" +} diff --git a/.kokoro/continuous/firestore.cfg b/.kokoro/continuous/firestore.cfg new file mode 100644 index 000000000000..9c5b0c53ab2d --- /dev/null +++ b/.kokoro/continuous/firestore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "firestore" +} diff --git a/.kokoro/continuous/grafeas.cfg b/.kokoro/continuous/grafeas.cfg new file mode 100644 index 000000000000..8f0e2f271518 --- /dev/null +++ b/.kokoro/continuous/grafeas.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "grafeas" +} diff --git a/.kokoro/continuous/iam.cfg b/.kokoro/continuous/iam.cfg new file mode 100644 index 000000000000..812d891c07f8 --- /dev/null +++ b/.kokoro/continuous/iam.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iam" +} diff --git a/.kokoro/continuous/iot.cfg b/.kokoro/continuous/iot.cfg new file mode 100644 index 000000000000..dd388a8323e7 --- /dev/null +++ b/.kokoro/continuous/iot.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iot" +} diff --git a/.kokoro/continuous/irm.cfg b/.kokoro/continuous/irm.cfg new file mode 100644 index 000000000000..ec847dd64e83 --- /dev/null +++ b/.kokoro/continuous/irm.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "irm" +} diff --git a/.kokoro/continuous/kms.cfg b/.kokoro/continuous/kms.cfg new file mode 100644 index 000000000000..ba634573085e --- /dev/null +++ b/.kokoro/continuous/kms.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "kms" +} diff --git a/.kokoro/continuous/language.cfg b/.kokoro/continuous/language.cfg new file mode 100644 index 000000000000..c7ee4e650ca3 --- /dev/null +++ b/.kokoro/continuous/language.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "language" +} diff --git a/.kokoro/continuous/logging.cfg b/.kokoro/continuous/logging.cfg new file mode 100644 index 000000000000..cf274e2d626b --- /dev/null +++ b/.kokoro/continuous/logging.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "logging" +} diff --git a/.kokoro/continuous/monitoring.cfg b/.kokoro/continuous/monitoring.cfg new file mode 100644 index 000000000000..80d261740e6f --- /dev/null +++ b/.kokoro/continuous/monitoring.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "monitoring" +} diff --git a/.kokoro/continuous/oslogin.cfg b/.kokoro/continuous/oslogin.cfg new file mode 100644 index 000000000000..9ca9766f6e3a --- /dev/null +++ b/.kokoro/continuous/oslogin.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "oslogin" +} diff --git a/.kokoro/continuous/phishingprotection.cfg b/.kokoro/continuous/phishingprotection.cfg new file mode 100644 index 000000000000..e21bd50ab4f1 --- /dev/null +++ b/.kokoro/continuous/phishingprotection.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "phishingprotection" +} diff --git a/.kokoro/continuous/pubsub.cfg b/.kokoro/continuous/pubsub.cfg new file mode 100644 index 000000000000..16c7ed3c175c --- /dev/null +++ b/.kokoro/continuous/pubsub.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "pubsub" +} diff --git a/.kokoro/continuous/redis.cfg b/.kokoro/continuous/redis.cfg new file mode 100644 index 000000000000..268bc06a7101 --- /dev/null +++ b/.kokoro/continuous/redis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "redis" +} diff --git a/.kokoro/continuous/resource_manager.cfg b/.kokoro/continuous/resource_manager.cfg new file mode 100644 index 000000000000..7a3976aef6cd --- /dev/null +++ b/.kokoro/continuous/resource_manager.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "resource_manager" +} diff --git a/.kokoro/continuous/runtimeconfig.cfg b/.kokoro/continuous/runtimeconfig.cfg new file mode 100644 index 000000000000..2900b3c7e91a --- /dev/null +++ b/.kokoro/continuous/runtimeconfig.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "runtimeconfig" +} diff --git a/.kokoro/continuous/scheduler.cfg b/.kokoro/continuous/scheduler.cfg new file mode 100644 index 000000000000..0701399853ef --- /dev/null +++ b/.kokoro/continuous/scheduler.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "scheduler" +} diff --git a/.kokoro/continuous/securitycenter.cfg b/.kokoro/continuous/securitycenter.cfg new file mode 100644 index 000000000000..17dc427dae15 --- /dev/null +++ b/.kokoro/continuous/securitycenter.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "securitycenter" +} diff --git a/.kokoro/continuous/spanner.cfg b/.kokoro/continuous/spanner.cfg new file mode 100644 index 000000000000..d986f9d137f9 --- /dev/null +++ b/.kokoro/continuous/spanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "spanner" +} diff --git a/.kokoro/continuous/speech.cfg b/.kokoro/continuous/speech.cfg new file mode 100644 index 000000000000..b178b790be8d --- /dev/null +++ b/.kokoro/continuous/speech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "speech" +} diff --git a/.kokoro/continuous/storage.cfg b/.kokoro/continuous/storage.cfg new file mode 100644 index 000000000000..6bd9371ff99d --- /dev/null +++ b/.kokoro/continuous/storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "storage" +} diff --git a/.kokoro/continuous/talent.cfg b/.kokoro/continuous/talent.cfg new file mode 100644 index 000000000000..abafead62769 --- /dev/null +++ b/.kokoro/continuous/talent.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "talent" +} diff --git a/.kokoro/continuous/tasks.cfg b/.kokoro/continuous/tasks.cfg new file mode 100644 index 000000000000..2ae5c581b88d --- /dev/null +++ b/.kokoro/continuous/tasks.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "tasks" +} diff --git a/.kokoro/continuous/texttospeech.cfg b/.kokoro/continuous/texttospeech.cfg new file mode 100644 index 000000000000..9e7d6d7f7980 --- /dev/null +++ b/.kokoro/continuous/texttospeech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "texttospeech" +} diff --git a/.kokoro/continuous/trace.cfg b/.kokoro/continuous/trace.cfg new file mode 100644 index 000000000000..9169296e4753 --- /dev/null +++ b/.kokoro/continuous/trace.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "trace" +} diff --git a/.kokoro/continuous/translate.cfg b/.kokoro/continuous/translate.cfg new file mode 100644 index 000000000000..807c96195b41 --- /dev/null +++ b/.kokoro/continuous/translate.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "translate" +} diff --git a/.kokoro/continuous/videointelligence.cfg b/.kokoro/continuous/videointelligence.cfg new file mode 100644 index 000000000000..77d1d3fc49e6 --- /dev/null +++ b/.kokoro/continuous/videointelligence.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "videointelligence" +} diff --git a/.kokoro/continuous/vision.cfg b/.kokoro/continuous/vision.cfg new file mode 100644 index 000000000000..4f686b3abf42 --- /dev/null +++ b/.kokoro/continuous/vision.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "vision" +} diff --git a/.kokoro/continuous/webrisk.cfg b/.kokoro/continuous/webrisk.cfg new file mode 100644 index 000000000000..36b06f0b0b55 --- /dev/null +++ b/.kokoro/continuous/webrisk.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "webrisk" +} diff --git a/.kokoro/continuous/websecurityscanner.cfg b/.kokoro/continuous/websecurityscanner.cfg new file mode 100644 index 000000000000..221d1fc23d6b --- /dev/null +++ b/.kokoro/continuous/websecurityscanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "websecurityscanner" +} diff --git a/.kokoro/docs/api_core.cfg b/.kokoro/docs/api_core.cfg new file mode 100644 index 000000000000..00ae35f46e2f --- /dev/null +++ b/.kokoro/docs/api_core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "api_core" +} diff --git a/.kokoro/docs/asset.cfg b/.kokoro/docs/asset.cfg new file mode 100644 index 000000000000..61c3fa9c5b4b --- /dev/null +++ b/.kokoro/docs/asset.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "asset" +} diff --git a/.kokoro/docs/automl.cfg b/.kokoro/docs/automl.cfg new file mode 100644 index 000000000000..093775c51065 --- /dev/null +++ b/.kokoro/docs/automl.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "automl" +} \ No newline at end of file diff --git a/.kokoro/docs/bigquery.cfg b/.kokoro/docs/bigquery.cfg new file mode 100644 index 000000000000..69eabb20803c --- /dev/null +++ b/.kokoro/docs/bigquery.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery" +} diff --git a/.kokoro/docs/bigquery_datatransfer.cfg b/.kokoro/docs/bigquery_datatransfer.cfg new file mode 100644 index 000000000000..47e510d21a44 --- /dev/null +++ b/.kokoro/docs/bigquery_datatransfer.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_datatransfer" +} diff --git a/.kokoro/docs/bigquery_storage.cfg b/.kokoro/docs/bigquery_storage.cfg new file mode 100644 index 000000000000..7f72bfed0587 --- /dev/null +++ b/.kokoro/docs/bigquery_storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_storage" +} diff --git a/.kokoro/docs/bigtable.cfg b/.kokoro/docs/bigtable.cfg new file mode 100644 index 000000000000..de67c4844de9 --- /dev/null +++ b/.kokoro/docs/bigtable.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigtable" +} diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg new file mode 100644 index 000000000000..96e9740dd889 --- /dev/null +++ b/.kokoro/docs/common.cfg @@ -0,0 +1,48 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "google-cloud-python/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-python/.kokoro/publish-docs.sh" +} + +env_vars: { + key: "STAGING_BUCKET" + value: "docs-staging" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} \ No newline at end of file diff --git a/.kokoro/docs/container.cfg b/.kokoro/docs/container.cfg new file mode 100644 index 000000000000..8d06b5285bf2 --- /dev/null +++ b/.kokoro/docs/container.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "container" +} diff --git a/.kokoro/docs/containeranalysis.cfg b/.kokoro/docs/containeranalysis.cfg new file mode 100644 index 000000000000..ae80ed8d8bc6 --- /dev/null +++ b/.kokoro/docs/containeranalysis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "containeranalysis" +} diff --git a/.kokoro/docs/core.cfg b/.kokoro/docs/core.cfg new file mode 100644 index 000000000000..68b0d504429c --- /dev/null +++ b/.kokoro/docs/core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "core" +} diff --git a/.kokoro/docs/datacatalog.cfg b/.kokoro/docs/datacatalog.cfg new file mode 100644 index 000000000000..fc0371e5eb11 --- /dev/null +++ b/.kokoro/docs/datacatalog.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datacatalog" +} diff --git a/.kokoro/docs/datalabeling.cfg b/.kokoro/docs/datalabeling.cfg new file mode 100644 index 000000000000..a1c53b81884b --- /dev/null +++ b/.kokoro/docs/datalabeling.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datalabeling" +} diff --git a/.kokoro/docs/dataproc.cfg b/.kokoro/docs/dataproc.cfg new file mode 100644 index 000000000000..d9ea2e670aea --- /dev/null +++ b/.kokoro/docs/dataproc.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dataproc" +} diff --git a/.kokoro/docs/datastore.cfg b/.kokoro/docs/datastore.cfg new file mode 100644 index 000000000000..7cf6a9c0dc8f --- /dev/null +++ b/.kokoro/docs/datastore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datastore" +} diff --git a/.kokoro/docs/dlp.cfg b/.kokoro/docs/dlp.cfg new file mode 100644 index 000000000000..931126d3e719 --- /dev/null +++ b/.kokoro/docs/dlp.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dlp" +} diff --git a/.kokoro/docs/dns.cfg b/.kokoro/docs/dns.cfg new file mode 100644 index 000000000000..845e3e148060 --- /dev/null +++ b/.kokoro/docs/dns.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dns" +} diff --git a/.kokoro/docs/error_reporting.cfg b/.kokoro/docs/error_reporting.cfg new file mode 100644 index 000000000000..99a86c522b9a --- /dev/null +++ b/.kokoro/docs/error_reporting.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "error_reporting" +} diff --git a/.kokoro/docs/firestore.cfg b/.kokoro/docs/firestore.cfg new file mode 100644 index 000000000000..9c5b0c53ab2d --- /dev/null +++ b/.kokoro/docs/firestore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "firestore" +} diff --git a/.kokoro/docs/grafeas.cfg b/.kokoro/docs/grafeas.cfg new file mode 100644 index 000000000000..8f0e2f271518 --- /dev/null +++ b/.kokoro/docs/grafeas.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "grafeas" +} diff --git a/.kokoro/docs/iam.cfg b/.kokoro/docs/iam.cfg new file mode 100644 index 000000000000..812d891c07f8 --- /dev/null +++ b/.kokoro/docs/iam.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iam" +} diff --git a/.kokoro/docs/iot.cfg b/.kokoro/docs/iot.cfg new file mode 100644 index 000000000000..114c8da20d77 --- /dev/null +++ b/.kokoro/docs/iot.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iot" +} \ No newline at end of file diff --git a/.kokoro/docs/irm.cfg b/.kokoro/docs/irm.cfg new file mode 100644 index 000000000000..ec847dd64e83 --- /dev/null +++ b/.kokoro/docs/irm.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "irm" +} diff --git a/.kokoro/docs/kms.cfg b/.kokoro/docs/kms.cfg new file mode 100644 index 000000000000..ba634573085e --- /dev/null +++ b/.kokoro/docs/kms.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "kms" +} diff --git a/.kokoro/docs/language.cfg b/.kokoro/docs/language.cfg new file mode 100644 index 000000000000..c7ee4e650ca3 --- /dev/null +++ b/.kokoro/docs/language.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "language" +} diff --git a/.kokoro/docs/logging.cfg b/.kokoro/docs/logging.cfg new file mode 100644 index 000000000000..cf274e2d626b --- /dev/null +++ b/.kokoro/docs/logging.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "logging" +} diff --git a/.kokoro/docs/monitoring.cfg b/.kokoro/docs/monitoring.cfg new file mode 100644 index 000000000000..80d261740e6f --- /dev/null +++ b/.kokoro/docs/monitoring.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "monitoring" +} diff --git a/.kokoro/docs/oslogin.cfg b/.kokoro/docs/oslogin.cfg new file mode 100644 index 000000000000..9ca9766f6e3a --- /dev/null +++ b/.kokoro/docs/oslogin.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "oslogin" +} diff --git a/.kokoro/docs/phishingprotection.cfg b/.kokoro/docs/phishingprotection.cfg new file mode 100644 index 000000000000..e21bd50ab4f1 --- /dev/null +++ b/.kokoro/docs/phishingprotection.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "phishingprotection" +} diff --git a/.kokoro/docs/pubsub.cfg b/.kokoro/docs/pubsub.cfg new file mode 100644 index 000000000000..16c7ed3c175c --- /dev/null +++ b/.kokoro/docs/pubsub.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "pubsub" +} diff --git a/.kokoro/docs/redis.cfg b/.kokoro/docs/redis.cfg new file mode 100644 index 000000000000..268bc06a7101 --- /dev/null +++ b/.kokoro/docs/redis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "redis" +} diff --git a/.kokoro/docs/resource_manager.cfg b/.kokoro/docs/resource_manager.cfg new file mode 100644 index 000000000000..7a3976aef6cd --- /dev/null +++ b/.kokoro/docs/resource_manager.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "resource_manager" +} diff --git a/.kokoro/docs/runtimeconfig.cfg b/.kokoro/docs/runtimeconfig.cfg new file mode 100644 index 000000000000..2900b3c7e91a --- /dev/null +++ b/.kokoro/docs/runtimeconfig.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "runtimeconfig" +} diff --git a/.kokoro/docs/scheduler.cfg b/.kokoro/docs/scheduler.cfg new file mode 100644 index 000000000000..0701399853ef --- /dev/null +++ b/.kokoro/docs/scheduler.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "scheduler" +} diff --git a/.kokoro/docs/securitycenter.cfg b/.kokoro/docs/securitycenter.cfg new file mode 100644 index 000000000000..17dc427dae15 --- /dev/null +++ b/.kokoro/docs/securitycenter.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "securitycenter" +} diff --git a/.kokoro/docs/spanner.cfg b/.kokoro/docs/spanner.cfg new file mode 100644 index 000000000000..d986f9d137f9 --- /dev/null +++ b/.kokoro/docs/spanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "spanner" +} diff --git a/.kokoro/docs/speech.cfg b/.kokoro/docs/speech.cfg new file mode 100644 index 000000000000..b178b790be8d --- /dev/null +++ b/.kokoro/docs/speech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "speech" +} diff --git a/.kokoro/docs/storage.cfg b/.kokoro/docs/storage.cfg new file mode 100644 index 000000000000..6bd9371ff99d --- /dev/null +++ b/.kokoro/docs/storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "storage" +} diff --git a/.kokoro/docs/talent.cfg b/.kokoro/docs/talent.cfg new file mode 100644 index 000000000000..abafead62769 --- /dev/null +++ b/.kokoro/docs/talent.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "talent" +} diff --git a/.kokoro/docs/tasks.cfg b/.kokoro/docs/tasks.cfg new file mode 100644 index 000000000000..2ae5c581b88d --- /dev/null +++ b/.kokoro/docs/tasks.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "tasks" +} diff --git a/.kokoro/docs/texttospeech.cfg b/.kokoro/docs/texttospeech.cfg new file mode 100644 index 000000000000..9e7d6d7f7980 --- /dev/null +++ b/.kokoro/docs/texttospeech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "texttospeech" +} diff --git a/.kokoro/docs/trace.cfg b/.kokoro/docs/trace.cfg new file mode 100644 index 000000000000..9169296e4753 --- /dev/null +++ b/.kokoro/docs/trace.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "trace" +} diff --git a/.kokoro/docs/translate.cfg b/.kokoro/docs/translate.cfg new file mode 100644 index 000000000000..807c96195b41 --- /dev/null +++ b/.kokoro/docs/translate.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "translate" +} diff --git a/.kokoro/docs/videointelligence.cfg b/.kokoro/docs/videointelligence.cfg new file mode 100644 index 000000000000..77d1d3fc49e6 --- /dev/null +++ b/.kokoro/docs/videointelligence.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "videointelligence" +} diff --git a/.kokoro/docs/vision.cfg b/.kokoro/docs/vision.cfg new file mode 100644 index 000000000000..4f686b3abf42 --- /dev/null +++ b/.kokoro/docs/vision.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "vision" +} diff --git a/.kokoro/docs/webrisk.cfg b/.kokoro/docs/webrisk.cfg new file mode 100644 index 000000000000..36b06f0b0b55 --- /dev/null +++ b/.kokoro/docs/webrisk.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "webrisk" +} diff --git a/.kokoro/docs/websecurityscanner.cfg b/.kokoro/docs/websecurityscanner.cfg new file mode 100644 index 000000000000..221d1fc23d6b --- /dev/null +++ b/.kokoro/docs/websecurityscanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "websecurityscanner" +} diff --git a/.kokoro/presubmit/api_core.cfg b/.kokoro/presubmit/api_core.cfg new file mode 100644 index 000000000000..00ae35f46e2f --- /dev/null +++ b/.kokoro/presubmit/api_core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "api_core" +} diff --git a/.kokoro/presubmit/asset.cfg b/.kokoro/presubmit/asset.cfg new file mode 100644 index 000000000000..61c3fa9c5b4b --- /dev/null +++ b/.kokoro/presubmit/asset.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "asset" +} diff --git a/.kokoro/presubmit/automl.cfg b/.kokoro/presubmit/automl.cfg new file mode 100644 index 000000000000..5081664abb56 --- /dev/null +++ b/.kokoro/presubmit/automl.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "automl" +} diff --git a/.kokoro/presubmit/bigquery.cfg b/.kokoro/presubmit/bigquery.cfg new file mode 100644 index 000000000000..69eabb20803c --- /dev/null +++ b/.kokoro/presubmit/bigquery.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery" +} diff --git a/.kokoro/presubmit/bigquery_datatransfer.cfg b/.kokoro/presubmit/bigquery_datatransfer.cfg new file mode 100644 index 000000000000..47e510d21a44 --- /dev/null +++ b/.kokoro/presubmit/bigquery_datatransfer.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_datatransfer" +} diff --git a/.kokoro/presubmit/bigquery_storage.cfg b/.kokoro/presubmit/bigquery_storage.cfg new file mode 100644 index 000000000000..7f72bfed0587 --- /dev/null +++ b/.kokoro/presubmit/bigquery_storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_storage" +} diff --git a/.kokoro/presubmit/bigtable.cfg b/.kokoro/presubmit/bigtable.cfg new file mode 100644 index 000000000000..de67c4844de9 --- /dev/null +++ b/.kokoro/presubmit/bigtable.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigtable" +} diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg new file mode 100644 index 000000000000..5dec06191476 --- /dev/null +++ b/.kokoro/presubmit/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "google-cloud-python/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-python/.kokoro/build.sh" +} diff --git a/.kokoro/presubmit/container.cfg b/.kokoro/presubmit/container.cfg new file mode 100644 index 000000000000..8d06b5285bf2 --- /dev/null +++ b/.kokoro/presubmit/container.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "container" +} diff --git a/.kokoro/presubmit/containeranalysis.cfg b/.kokoro/presubmit/containeranalysis.cfg new file mode 100644 index 000000000000..ae80ed8d8bc6 --- /dev/null +++ b/.kokoro/presubmit/containeranalysis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "containeranalysis" +} diff --git a/.kokoro/presubmit/core.cfg b/.kokoro/presubmit/core.cfg new file mode 100644 index 000000000000..68b0d504429c --- /dev/null +++ b/.kokoro/presubmit/core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "core" +} diff --git a/.kokoro/presubmit/datacatalog.cfg b/.kokoro/presubmit/datacatalog.cfg new file mode 100644 index 000000000000..fc0371e5eb11 --- /dev/null +++ b/.kokoro/presubmit/datacatalog.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datacatalog" +} diff --git a/.kokoro/presubmit/datalabeling.cfg b/.kokoro/presubmit/datalabeling.cfg new file mode 100644 index 000000000000..a1c53b81884b --- /dev/null +++ b/.kokoro/presubmit/datalabeling.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datalabeling" +} diff --git a/.kokoro/presubmit/dataproc.cfg b/.kokoro/presubmit/dataproc.cfg new file mode 100644 index 000000000000..d9ea2e670aea --- /dev/null +++ b/.kokoro/presubmit/dataproc.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dataproc" +} diff --git a/.kokoro/presubmit/datastore.cfg b/.kokoro/presubmit/datastore.cfg new file mode 100644 index 000000000000..7cf6a9c0dc8f --- /dev/null +++ b/.kokoro/presubmit/datastore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datastore" +} diff --git a/.kokoro/presubmit/dlp.cfg b/.kokoro/presubmit/dlp.cfg new file mode 100644 index 000000000000..931126d3e719 --- /dev/null +++ b/.kokoro/presubmit/dlp.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dlp" +} diff --git a/.kokoro/presubmit/dns.cfg b/.kokoro/presubmit/dns.cfg new file mode 100644 index 000000000000..845e3e148060 --- /dev/null +++ b/.kokoro/presubmit/dns.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dns" +} diff --git a/.kokoro/presubmit/error_reporting.cfg b/.kokoro/presubmit/error_reporting.cfg new file mode 100644 index 000000000000..99a86c522b9a --- /dev/null +++ b/.kokoro/presubmit/error_reporting.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "error_reporting" +} diff --git a/.kokoro/presubmit/firestore.cfg b/.kokoro/presubmit/firestore.cfg new file mode 100644 index 000000000000..9c5b0c53ab2d --- /dev/null +++ b/.kokoro/presubmit/firestore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "firestore" +} diff --git a/.kokoro/presubmit/grafeas.cfg b/.kokoro/presubmit/grafeas.cfg new file mode 100644 index 000000000000..8f0e2f271518 --- /dev/null +++ b/.kokoro/presubmit/grafeas.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "grafeas" +} diff --git a/.kokoro/presubmit/iam.cfg b/.kokoro/presubmit/iam.cfg new file mode 100644 index 000000000000..812d891c07f8 --- /dev/null +++ b/.kokoro/presubmit/iam.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iam" +} diff --git a/.kokoro/presubmit/iot.cfg b/.kokoro/presubmit/iot.cfg new file mode 100644 index 000000000000..dd388a8323e7 --- /dev/null +++ b/.kokoro/presubmit/iot.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iot" +} diff --git a/.kokoro/presubmit/irm.cfg b/.kokoro/presubmit/irm.cfg new file mode 100644 index 000000000000..ec847dd64e83 --- /dev/null +++ b/.kokoro/presubmit/irm.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "irm" +} diff --git a/.kokoro/presubmit/kms.cfg b/.kokoro/presubmit/kms.cfg new file mode 100644 index 000000000000..ba634573085e --- /dev/null +++ b/.kokoro/presubmit/kms.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "kms" +} diff --git a/.kokoro/presubmit/language.cfg b/.kokoro/presubmit/language.cfg new file mode 100644 index 000000000000..c7ee4e650ca3 --- /dev/null +++ b/.kokoro/presubmit/language.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "language" +} diff --git a/.kokoro/presubmit/logging.cfg b/.kokoro/presubmit/logging.cfg new file mode 100644 index 000000000000..cf274e2d626b --- /dev/null +++ b/.kokoro/presubmit/logging.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "logging" +} diff --git a/.kokoro/presubmit/monitoring.cfg b/.kokoro/presubmit/monitoring.cfg new file mode 100644 index 000000000000..80d261740e6f --- /dev/null +++ b/.kokoro/presubmit/monitoring.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "monitoring" +} diff --git a/.kokoro/presubmit/oslogin.cfg b/.kokoro/presubmit/oslogin.cfg new file mode 100644 index 000000000000..9ca9766f6e3a --- /dev/null +++ b/.kokoro/presubmit/oslogin.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "oslogin" +} diff --git a/.kokoro/presubmit/phishingprotection.cfg b/.kokoro/presubmit/phishingprotection.cfg new file mode 100644 index 000000000000..e21bd50ab4f1 --- /dev/null +++ b/.kokoro/presubmit/phishingprotection.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "phishingprotection" +} diff --git a/.kokoro/presubmit/pubsub.cfg b/.kokoro/presubmit/pubsub.cfg new file mode 100644 index 000000000000..16c7ed3c175c --- /dev/null +++ b/.kokoro/presubmit/pubsub.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "pubsub" +} diff --git a/.kokoro/presubmit/redis.cfg b/.kokoro/presubmit/redis.cfg new file mode 100644 index 000000000000..268bc06a7101 --- /dev/null +++ b/.kokoro/presubmit/redis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "redis" +} diff --git a/.kokoro/presubmit/resource_manager.cfg b/.kokoro/presubmit/resource_manager.cfg new file mode 100644 index 000000000000..7a3976aef6cd --- /dev/null +++ b/.kokoro/presubmit/resource_manager.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "resource_manager" +} diff --git a/.kokoro/presubmit/runtimeconfig.cfg b/.kokoro/presubmit/runtimeconfig.cfg new file mode 100644 index 000000000000..2900b3c7e91a --- /dev/null +++ b/.kokoro/presubmit/runtimeconfig.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "runtimeconfig" +} diff --git a/.kokoro/presubmit/scheduler.cfg b/.kokoro/presubmit/scheduler.cfg new file mode 100644 index 000000000000..0701399853ef --- /dev/null +++ b/.kokoro/presubmit/scheduler.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "scheduler" +} diff --git a/.kokoro/presubmit/securitycenter.cfg b/.kokoro/presubmit/securitycenter.cfg new file mode 100644 index 000000000000..17dc427dae15 --- /dev/null +++ b/.kokoro/presubmit/securitycenter.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "securitycenter" +} diff --git a/.kokoro/presubmit/spanner.cfg b/.kokoro/presubmit/spanner.cfg new file mode 100644 index 000000000000..d986f9d137f9 --- /dev/null +++ b/.kokoro/presubmit/spanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "spanner" +} diff --git a/.kokoro/presubmit/speech.cfg b/.kokoro/presubmit/speech.cfg new file mode 100644 index 000000000000..b178b790be8d --- /dev/null +++ b/.kokoro/presubmit/speech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "speech" +} diff --git a/.kokoro/presubmit/storage.cfg b/.kokoro/presubmit/storage.cfg new file mode 100644 index 000000000000..6bd9371ff99d --- /dev/null +++ b/.kokoro/presubmit/storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "storage" +} diff --git a/.kokoro/presubmit/talent.cfg b/.kokoro/presubmit/talent.cfg new file mode 100644 index 000000000000..abafead62769 --- /dev/null +++ b/.kokoro/presubmit/talent.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "talent" +} diff --git a/.kokoro/presubmit/tasks.cfg b/.kokoro/presubmit/tasks.cfg new file mode 100644 index 000000000000..2ae5c581b88d --- /dev/null +++ b/.kokoro/presubmit/tasks.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "tasks" +} diff --git a/.kokoro/presubmit/texttospeech.cfg b/.kokoro/presubmit/texttospeech.cfg new file mode 100644 index 000000000000..9e7d6d7f7980 --- /dev/null +++ b/.kokoro/presubmit/texttospeech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "texttospeech" +} diff --git a/.kokoro/presubmit/trace.cfg b/.kokoro/presubmit/trace.cfg new file mode 100644 index 000000000000..9169296e4753 --- /dev/null +++ b/.kokoro/presubmit/trace.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "trace" +} diff --git a/.kokoro/presubmit/translate.cfg b/.kokoro/presubmit/translate.cfg new file mode 100644 index 000000000000..807c96195b41 --- /dev/null +++ b/.kokoro/presubmit/translate.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "translate" +} diff --git a/.kokoro/presubmit/videointelligence.cfg b/.kokoro/presubmit/videointelligence.cfg new file mode 100644 index 000000000000..77d1d3fc49e6 --- /dev/null +++ b/.kokoro/presubmit/videointelligence.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "videointelligence" +} diff --git a/.kokoro/presubmit/vision.cfg b/.kokoro/presubmit/vision.cfg new file mode 100644 index 000000000000..4f686b3abf42 --- /dev/null +++ b/.kokoro/presubmit/vision.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "vision" +} diff --git a/.kokoro/presubmit/webrisk.cfg b/.kokoro/presubmit/webrisk.cfg new file mode 100644 index 000000000000..36b06f0b0b55 --- /dev/null +++ b/.kokoro/presubmit/webrisk.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "webrisk" +} diff --git a/.kokoro/presubmit/websecurityscanner.cfg b/.kokoro/presubmit/websecurityscanner.cfg new file mode 100644 index 000000000000..221d1fc23d6b --- /dev/null +++ b/.kokoro/presubmit/websecurityscanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "websecurityscanner" +} diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh new file mode 100755 index 000000000000..d6762b5ed6f9 --- /dev/null +++ b/.kokoro/publish-docs.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -eo pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +cd github/google-cloud-python/${PACKAGE} + +# Remove old nox +python3.6 -m pip uninstall --yes --quiet nox-automation + +# Install nox +python3.6 -m pip install --upgrade --quiet nox +python3.6 -m nox --version + +# build docs +nox -s docs + +python3 -m pip install gcp-docuploader + +# install a json parser +sudo apt-get update +sudo apt-get -y install software-properties-common +sudo add-apt-repository universe +sudo apt-get update +sudo apt-get -y install jq + +# create metadata +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging \ No newline at end of file diff --git a/.kokoro/release.sh b/.kokoro/release.sh new file mode 100755 index 000000000000..6909ae880d8b --- /dev/null +++ b/.kokoro/release.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eo pipefail + +# Start the releasetool reporter +python3 -m pip install gcp-releasetool +python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. +python3 -m pip install --upgrade twine wheel setuptools + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Move into the package, build the distribution and upload. +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password") +cd github/google-cloud-python/${PACKAGE} +python3 setup.py sdist bdist_wheel +twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/api_core.cfg b/.kokoro/release/api_core.cfg new file mode 100644 index 000000000000..00ae35f46e2f --- /dev/null +++ b/.kokoro/release/api_core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "api_core" +} diff --git a/.kokoro/release/asset.cfg b/.kokoro/release/asset.cfg new file mode 100644 index 000000000000..61c3fa9c5b4b --- /dev/null +++ b/.kokoro/release/asset.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "asset" +} diff --git a/.kokoro/release/automl.cfg b/.kokoro/release/automl.cfg new file mode 100644 index 000000000000..5081664abb56 --- /dev/null +++ b/.kokoro/release/automl.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "automl" +} diff --git a/.kokoro/release/bigquery.cfg b/.kokoro/release/bigquery.cfg new file mode 100644 index 000000000000..69eabb20803c --- /dev/null +++ b/.kokoro/release/bigquery.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery" +} diff --git a/.kokoro/release/bigquery_datatransfer.cfg b/.kokoro/release/bigquery_datatransfer.cfg new file mode 100644 index 000000000000..47e510d21a44 --- /dev/null +++ b/.kokoro/release/bigquery_datatransfer.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_datatransfer" +} diff --git a/.kokoro/release/bigquery_storage.cfg b/.kokoro/release/bigquery_storage.cfg new file mode 100644 index 000000000000..7f72bfed0587 --- /dev/null +++ b/.kokoro/release/bigquery_storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigquery_storage" +} diff --git a/.kokoro/release/bigtable.cfg b/.kokoro/release/bigtable.cfg new file mode 100644 index 000000000000..de67c4844de9 --- /dev/null +++ b/.kokoro/release/bigtable.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "bigtable" +} diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg new file mode 100644 index 000000000000..7456ab5604ef --- /dev/null +++ b/.kokoro/release/common.cfg @@ -0,0 +1,64 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "google-cloud-python/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-python/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google_cloud_pypi_password" + } + } +} + +# Fetch magictoken to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "releasetool-magictoken" + } + } +} + +# Fetch api key to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "magic-github-proxy-api-key" + } + } +} diff --git a/.kokoro/release/container.cfg b/.kokoro/release/container.cfg new file mode 100644 index 000000000000..8d06b5285bf2 --- /dev/null +++ b/.kokoro/release/container.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "container" +} diff --git a/.kokoro/release/containeranalysis.cfg b/.kokoro/release/containeranalysis.cfg new file mode 100644 index 000000000000..ae80ed8d8bc6 --- /dev/null +++ b/.kokoro/release/containeranalysis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "containeranalysis" +} diff --git a/.kokoro/release/core.cfg b/.kokoro/release/core.cfg new file mode 100644 index 000000000000..68b0d504429c --- /dev/null +++ b/.kokoro/release/core.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "core" +} diff --git a/.kokoro/release/datacatalog.cfg b/.kokoro/release/datacatalog.cfg new file mode 100644 index 000000000000..fc0371e5eb11 --- /dev/null +++ b/.kokoro/release/datacatalog.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datacatalog" +} diff --git a/.kokoro/release/datalabeling.cfg b/.kokoro/release/datalabeling.cfg new file mode 100644 index 000000000000..a1c53b81884b --- /dev/null +++ b/.kokoro/release/datalabeling.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datalabeling" +} diff --git a/.kokoro/release/dataproc.cfg b/.kokoro/release/dataproc.cfg new file mode 100644 index 000000000000..d9ea2e670aea --- /dev/null +++ b/.kokoro/release/dataproc.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dataproc" +} diff --git a/.kokoro/release/datastore.cfg b/.kokoro/release/datastore.cfg new file mode 100644 index 000000000000..7cf6a9c0dc8f --- /dev/null +++ b/.kokoro/release/datastore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "datastore" +} diff --git a/.kokoro/release/dlp.cfg b/.kokoro/release/dlp.cfg new file mode 100644 index 000000000000..931126d3e719 --- /dev/null +++ b/.kokoro/release/dlp.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dlp" +} diff --git a/.kokoro/release/dns.cfg b/.kokoro/release/dns.cfg new file mode 100644 index 000000000000..845e3e148060 --- /dev/null +++ b/.kokoro/release/dns.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "dns" +} diff --git a/.kokoro/release/error_reporting.cfg b/.kokoro/release/error_reporting.cfg new file mode 100644 index 000000000000..99a86c522b9a --- /dev/null +++ b/.kokoro/release/error_reporting.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "error_reporting" +} diff --git a/.kokoro/release/firestore.cfg b/.kokoro/release/firestore.cfg new file mode 100644 index 000000000000..9c5b0c53ab2d --- /dev/null +++ b/.kokoro/release/firestore.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "firestore" +} diff --git a/.kokoro/release/grafeas.cfg b/.kokoro/release/grafeas.cfg new file mode 100644 index 000000000000..8f0e2f271518 --- /dev/null +++ b/.kokoro/release/grafeas.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "grafeas" +} diff --git a/.kokoro/release/iam.cfg b/.kokoro/release/iam.cfg new file mode 100644 index 000000000000..812d891c07f8 --- /dev/null +++ b/.kokoro/release/iam.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iam" +} diff --git a/.kokoro/release/iot.cfg b/.kokoro/release/iot.cfg new file mode 100644 index 000000000000..dd388a8323e7 --- /dev/null +++ b/.kokoro/release/iot.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "iot" +} diff --git a/.kokoro/release/irm.cfg b/.kokoro/release/irm.cfg new file mode 100644 index 000000000000..ec847dd64e83 --- /dev/null +++ b/.kokoro/release/irm.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "irm" +} diff --git a/.kokoro/release/kms.cfg b/.kokoro/release/kms.cfg new file mode 100644 index 000000000000..ba634573085e --- /dev/null +++ b/.kokoro/release/kms.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "kms" +} diff --git a/.kokoro/release/language.cfg b/.kokoro/release/language.cfg new file mode 100644 index 000000000000..c7ee4e650ca3 --- /dev/null +++ b/.kokoro/release/language.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "language" +} diff --git a/.kokoro/release/logging.cfg b/.kokoro/release/logging.cfg new file mode 100644 index 000000000000..cf274e2d626b --- /dev/null +++ b/.kokoro/release/logging.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "logging" +} diff --git a/.kokoro/release/monitoring.cfg b/.kokoro/release/monitoring.cfg new file mode 100644 index 000000000000..80d261740e6f --- /dev/null +++ b/.kokoro/release/monitoring.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "monitoring" +} diff --git a/.kokoro/release/oslogin.cfg b/.kokoro/release/oslogin.cfg new file mode 100644 index 000000000000..9ca9766f6e3a --- /dev/null +++ b/.kokoro/release/oslogin.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "oslogin" +} diff --git a/.kokoro/release/phishingprotection.cfg b/.kokoro/release/phishingprotection.cfg new file mode 100644 index 000000000000..e21bd50ab4f1 --- /dev/null +++ b/.kokoro/release/phishingprotection.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "phishingprotection" +} diff --git a/.kokoro/release/pubsub.cfg b/.kokoro/release/pubsub.cfg new file mode 100644 index 000000000000..16c7ed3c175c --- /dev/null +++ b/.kokoro/release/pubsub.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "pubsub" +} diff --git a/.kokoro/release/redis.cfg b/.kokoro/release/redis.cfg new file mode 100644 index 000000000000..268bc06a7101 --- /dev/null +++ b/.kokoro/release/redis.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "redis" +} diff --git a/.kokoro/release/resource_manager.cfg b/.kokoro/release/resource_manager.cfg new file mode 100644 index 000000000000..7a3976aef6cd --- /dev/null +++ b/.kokoro/release/resource_manager.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "resource_manager" +} diff --git a/.kokoro/release/runtimeconfig.cfg b/.kokoro/release/runtimeconfig.cfg new file mode 100644 index 000000000000..2900b3c7e91a --- /dev/null +++ b/.kokoro/release/runtimeconfig.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "runtimeconfig" +} diff --git a/.kokoro/release/scheduler.cfg b/.kokoro/release/scheduler.cfg new file mode 100644 index 000000000000..0701399853ef --- /dev/null +++ b/.kokoro/release/scheduler.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "scheduler" +} diff --git a/.kokoro/release/securitycenter.cfg b/.kokoro/release/securitycenter.cfg new file mode 100644 index 000000000000..17dc427dae15 --- /dev/null +++ b/.kokoro/release/securitycenter.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "securitycenter" +} diff --git a/.kokoro/release/spanner.cfg b/.kokoro/release/spanner.cfg new file mode 100644 index 000000000000..d986f9d137f9 --- /dev/null +++ b/.kokoro/release/spanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "spanner" +} diff --git a/.kokoro/release/speech.cfg b/.kokoro/release/speech.cfg new file mode 100644 index 000000000000..b178b790be8d --- /dev/null +++ b/.kokoro/release/speech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "speech" +} diff --git a/.kokoro/release/storage.cfg b/.kokoro/release/storage.cfg new file mode 100644 index 000000000000..6bd9371ff99d --- /dev/null +++ b/.kokoro/release/storage.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "storage" +} diff --git a/.kokoro/release/talent.cfg b/.kokoro/release/talent.cfg new file mode 100644 index 000000000000..abafead62769 --- /dev/null +++ b/.kokoro/release/talent.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "talent" +} diff --git a/.kokoro/release/tasks.cfg b/.kokoro/release/tasks.cfg new file mode 100644 index 000000000000..2ae5c581b88d --- /dev/null +++ b/.kokoro/release/tasks.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "tasks" +} diff --git a/.kokoro/release/texttospeech.cfg b/.kokoro/release/texttospeech.cfg new file mode 100644 index 000000000000..9e7d6d7f7980 --- /dev/null +++ b/.kokoro/release/texttospeech.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "texttospeech" +} diff --git a/.kokoro/release/trace.cfg b/.kokoro/release/trace.cfg new file mode 100644 index 000000000000..9169296e4753 --- /dev/null +++ b/.kokoro/release/trace.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "trace" +} diff --git a/.kokoro/release/translate.cfg b/.kokoro/release/translate.cfg new file mode 100644 index 000000000000..807c96195b41 --- /dev/null +++ b/.kokoro/release/translate.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "translate" +} diff --git a/.kokoro/release/videointelligence.cfg b/.kokoro/release/videointelligence.cfg new file mode 100644 index 000000000000..77d1d3fc49e6 --- /dev/null +++ b/.kokoro/release/videointelligence.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "videointelligence" +} diff --git a/.kokoro/release/vision.cfg b/.kokoro/release/vision.cfg new file mode 100644 index 000000000000..4f686b3abf42 --- /dev/null +++ b/.kokoro/release/vision.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "vision" +} diff --git a/.kokoro/release/webrisk.cfg b/.kokoro/release/webrisk.cfg new file mode 100644 index 000000000000..36b06f0b0b55 --- /dev/null +++ b/.kokoro/release/webrisk.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "webrisk" +} diff --git a/.kokoro/release/websecurityscanner.cfg b/.kokoro/release/websecurityscanner.cfg new file mode 100644 index 000000000000..221d1fc23d6b --- /dev/null +++ b/.kokoro/release/websecurityscanner.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "websecurityscanner" +} diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh new file mode 100755 index 000000000000..e8c4251f3ed4 --- /dev/null +++ b/.kokoro/trampoline.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$? + +chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh +${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true + +exit ${ret_code} diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000000..607de497679a --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,5 @@ +requirements_file: docs/requirements.txt +build: + image: latest +python: + version: 3.6 diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000000..86179f70d44f --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//bigtable/docs/conf.py=utf-8 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b222923baf11..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -language: python -sudo: false - -install: - - pip install --upgrade pip tox - -script: - - tox -e py27 - - tox -e py34 - - tox -e lint - - tox -e cover - - tox -e system-tests - - tox -e system-tests3 - - scripts/update_docs.sh - -after_success: - - tox -e coveralls - -deploy: - provider: pypi - user: gcloudpypi - password: - secure: LR0i9Oeu6kpLTYS5xK/zCng4gmdtPvFfD/XYdQhyY5jBibQkC2WUQU6nJA9bDXRxhBP5bUwXFGkbhOcOJgHNrUfmyPzpDbM8BR29KfY0WfdYv72gsGZOaekqCReFmHbqLE7qOQtHR5U3ey6ivcgw+hZO72Uu6qDCc9B8qwoBfAs= - on: - tags: true - repo: GoogleCloudPlatform/google-cloud-python - all_branches: true - # 'bdist_wheel' builds disabled until #1879 et al. are resolved. - distributions: "sdist" - -cache: - directories: - - ${HOME}/.cache/pip diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000000..7301a860a45c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,73 @@ +# Changelog + +[PyPI History][1] + +[1]: https://pypi.org/project/google-cloud/#history + +## 0.32.0 + +Packages newly added to the `google-cloud` umbrella package: + +- [`google-cloud-bigquery-datatransfer==0.1.0`](https://pypi.org/project/google-cloud-bigquery-datatransfer/0.1.0/) +- [`google-cloud-container==0.1.0`](https://pypi.org/project/google-cloud-container/0.1.0/) + +Feature / bugfix release for the following packages: + +- [`google-api-core==0.1.2`](https://pypi.org/project/google-api-core/0.1.2/) +- [`google-cloud-bigtable==0.28.1`](https://pypi.org/project/google-cloud-bigtable/0.28.1/) +- [`google-cloud-pubsub==0.30.0`](https://pypi.org/project/google-cloud-pubsub/0.30.0/) +- [`google-cloud-trace==0.17.0`](https://pypi.org/project/google-cloud-trace/0.17.0/) +- [`google-cloud-vision==0.29.0`](https://pypi.org/project/google-cloud-vision/0.29.0/) + +## 0.31.0 + +Feature / bugfix release for the following packages: + + * [`google-cloud-pubsub==0.29.1`](https://pypi.org/project/google-cloud-pubsub/0.29.1/) + * [`google-cloud-videointelligence==1.0.0`](https://pypi.org/project/google-cloud-videointelligence/1.0.0/) + +## 0.30.0 + +Feature / bugfix release for the following package: + + * [`google-cloud-language==1.0.0`](https://pypi.org/project/google-cloud-language/1.0.0/) + +## 0.29.0 + +Feature / bugfix release for the following package: + +- [`google-cloud-language==0.31.0`](https://pypi.org/project/google-cloud-language/0.31.0/) + +## 0.28.0 + +Packages newly added to the `google-cloud` umbrella package: + +- [`google-api-core==0.1.1`](https://pypi.org/project/google-api-core/0.1.1/) +- [`google-cloud-firestore==0.28.0`](https://pypi.org/project/google-cloud-firestore/0.28.0/) +- [`google-cloud-trace==0.16.0`](https://pypi.org/project/google-cloud-trace/0.16.0/) +- [`google-cloud-videointelligence=0.28.0`](https://pypi.org/project/google-cloud-videointelligence/0.28.0/) + +Feature / bugfix releases for the following packages: + +- [`google-cloud-bigquery==0.28.0`](https://pypi.org/project/google-cloud-bigquery/0.28.0/) +- [`google-cloud-core==0.28.0`](https://pypi.org/project/google-cloud-core/0.28.0/) +- [`google-cloud-datastore==1.4.0`](https://pypi.org/project/google-cloud-datastore/1.4.0/) +- [`google-cloud-logging==1.4.0`](https://pypi.org/project/google-cloud-logging/1.4.0/) +- [`google-cloud-pubsub==0.29.0`](https://pypi.org/project/google-cloud-pubsub/0.29.0/) +- [`google-cloud-spanner==0.29.0`](https://pypi.org/project/google-cloud-spanner/0.29.0/) +- [`google-cloud-translate==1.3.0`](https://pypi.org/project/google-cloud-translate/1.3.0/) +- [`google-cloud-vision==0.28.0`](https://pypi.org/project/google-cloud-vision/0.28.0/) + +Dependency-only / user-invisible changes to the following packages: + +- [`google-cloud-bigtable==0.28.0`](https://pypi.org/project/google-cloud-bigtable/0.28.0/) +- [`google-cloud-dns==0.28.0`](https://pypi.org/project/google-cloud-dns/0.28.0/) +- [`google-cloud-error-reporting==0.28.0`](https://pypi.org/project/google-cloud-error-reporting/0.28.0/) +- [`google-cloud-language==0.30.0`](https://pypi.org/project/google-cloud-language/0.30.0/) +- [`google-cloud-monitoring==0.28.0`](https://pypi.org/project/google-cloud-monitoring/0.28.0/) +- [`google-cloud-resource-manager==0.28.0`](https://pypi.org/project/google-cloud-resource-manager/0.28.0/) +- [`google-cloud-runtimeconfig==0.28.0`](https://pypi.org/project/google-cloud-runtimeconfig/0.28.0/) +- [`google-cloud-speech==0.30.0`](https://pypi.org/project/google-cloud-speech/0.30.0/) +- [`google-cloud-storage==1.6.0`](https://pypi.org/project/google-cloud-storage/1.6.0/) + +PyPI: https://pypi.org/project/google-cloud/0.28.0/ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9cb07618962e..b05875c89e81 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,37 +1,55 @@ +############ Contributing -============================ +############ #. **Please sign one of the contributor license agreements below.** #. Fork the repo, develop and test your code changes, add docs. #. Make sure that your commit messages clearly describe the changes. -#. Send a pull request. +#. Send a pull request. (Please Read: `Faster Pull Request Reviews`_) -Here are some guidelines for hacking on ``google-cloud-python``. +.. _Faster Pull Request Reviews: https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews -Using a Development Checkout ----------------------------- +.. contents:: Here are some guidelines for hacking on ``google-cloud-python``. + +*************** +Adding Features +*************** + +In order to add a feature to ``google-cloud-python``: + +- The feature must be documented in both the API and narrative + documentation (in ``docs/``). + +- The feature must work fully on the following CPython versions: 2.7, + 3.5, 3.6, and 3.7 on both UNIX and Windows. -You'll have to create a development environment to hack on ``google-cloud-python``, -using a Git checkout: +- The feature must not add unnecessary dependencies (where + "unnecessary" is of course subjective, but new dependencies should + be discussed). -- While logged into your GitHub account, navigate to the ``google-cloud-python`` repo - on GitHub. +**************************** +Using a Development Checkout +**************************** - https://github.com/GoogleCloudPlatform/google-cloud-python +You'll have to create a development environment to hack on +``google-cloud-python``, using a Git checkout: + +- While logged into your GitHub account, navigate to the + ``google-cloud-python`` `repo`_ on GitHub. - Fork and clone the ``google-cloud-python`` repository to your GitHub account by clicking the "Fork" button. - Clone your fork of ``google-cloud-python`` from your GitHub account to your local computer, substituting your account username and specifying the destination - as "hack-on-google-cloud-python". E.g.:: + as ``hack-on-google-cloud-python``. E.g.:: - $ cd ~ + $ cd ${HOME} $ git clone git@github.com:USERNAME/google-cloud-python.git hack-on-google-cloud-python $ cd hack-on-google-cloud-python # Configure remotes such that you can pull changes from the google-cloud-python # repository into your local repository. - $ git remote add upstream https://github.com:GoogleCloudPlatform/google-cloud-python + $ git remote add upstream git@github.com:GoogleCloudPlatform/google-cloud-python.git # fetch and merge changes from upstream into master $ git fetch upstream $ git merge upstream/master @@ -39,69 +57,82 @@ using a Git checkout: Now your local repo is set up such that you will push changes to your GitHub repo, from which you can submit a pull request. -- Create a virtualenv in which to install ``google-cloud-python``:: +To work on the codebase and run the tests, we recommend using ``nox``, +but you can also use a ``virtualenv`` of your own creation. - $ cd ~/hack-on-google-cloud-python - $ virtualenv --python python2.7 env +.. _repo: https://github.com/GoogleCloudPlatform/google-cloud-python - Note that very old versions of virtualenv (virtualenv versions below, say, - 1.10 or thereabouts) require you to pass a ``--no-site-packages`` flag to - get a completely isolated environment. +Using ``nox`` +============= - You can choose which Python version you want to use by passing a ``--python`` - flag to ``virtualenv``. For example, ``virtualenv --python python2.7`` - chooses the Python 2.7 interpreter to be installed. +We use `nox `__ to instrument our tests. - From here on in within these instructions, the ``~/hack-on-google-cloud-python/env`` - virtual environment you created above will be referred to as ``$VENV``. - To use the instructions in the steps that follow literally, use the - ``export VENV=~/hack-on-google-cloud-python/env`` command. +- To test your changes, run unit tests with ``nox``:: -- Install ``google-cloud-python`` from the checkout into the virtualenv using - ``setup.py develop``. Running ``setup.py develop`` *must* be done while - the current working directory is the ``google-cloud-python`` checkout directory:: + $ nox -f datastore/noxfile.py -s unit-2.7 + $ nox -f datastore/noxfile.py -s unit-3.7 + $ ... - $ cd ~/hack-on-google-cloud-python - $ $VENV/bin/python setup.py develop + .. note:: -I'm getting weird errors... Can you help? ------------------------------------------ + The unit tests and system tests are contained in the individual + ``nox.py`` files in each directory; substitute ``datastore`` in the + example above with the package of your choice. -If the error mentions ``Python.h`` not being found, -install ``python-dev`` and try again. -On Debian/Ubuntu:: - $ sudo apt-get install python-dev + Alternatively, you can just navigate directly to the package you are + currently developing and run tests there:: -Adding Features ---------------- + $ export GIT_ROOT=$(pwd) + $ cd ${GIT_ROOT}/datastore/ + $ nox -s "unit(py='3.7')" -In order to add a feature to ``google-cloud-python``: +.. nox: https://pypi.org/project/nox-automation/ -- The feature must be documented in both the API and narrative - documentation (in ``docs/``). +Note on Editable Installs / Develop Mode +======================================== -- The feature must work fully on the following CPython versions: 2.7, - 3.4, and 3.5 on both UNIX and Windows. +- As mentioned previously, using ``setuptools`` in `develop mode`_ + or a ``pip`` `editable install`_ is not possible with this + library. This is because this library uses `namespace packages`_. + For context see `Issue #2316`_ and the relevant `PyPA issue`_. -- The feature must not add unnecessary dependencies (where - "unnecessary" is of course subjective, but new dependencies should - be discussed). + Since ``editable`` / ``develop`` mode can't be used, packages + need to be installed directly. Hence your changes to the source + tree don't get incorporated into the **already installed** + package. + +.. _namespace packages: https://www.python.org/dev/peps/pep-0420/ +.. _Issue #2316: https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2316 +.. _PyPA issue: https://github.com/pypa/packaging-problems/issues/12 +.. _develop mode: https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode +.. _editable install: https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs +***************************************** +I'm getting weird errors... Can you help? +***************************************** + +If the error mentions ``Python.h`` not being found, +install ``python-dev`` and try again. +On Debian/Ubuntu:: + + $ sudo apt-get install python-dev + +************ Coding Style ------------- +************ -- PEP8 compliance, with exceptions defined in ``tox.ini``. - If you have ``tox`` installed, you can test that you have not introduced +- PEP8 compliance, with exceptions defined in the linter configuration. + If you have ``nox`` installed, you can test that you have not introduced any non-compliant code via:: - $ tox -e lint + $ nox -s lint -- In order to make ``tox -e lint`` run faster, you can set some environment +- In order to make ``nox -s lint`` run faster, you can set some environment variables:: - export GOOGLE_CLOUD_REMOTE_FOR_LINT="upstream" - export GOOGLE_CLOUD_BRANCH_FOR_LINT="master" + export GOOGLE_CLOUD_TESTING_REMOTE="upstream" + export GOOGLE_CLOUD_TESTING_BRANCH="master" By doing this, you are specifying the location of the most up-to-date version of ``google-cloud-python``. The the suggested remote name ``upstream`` @@ -110,47 +141,24 @@ Coding Style Exceptions to PEP8: -- Many unit tests use a helper method, ``_callFUT`` ("FUT" is short for +- Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for "Function-Under-Test"), which is PEP8-incompliant, but more readable. Some also use a local variable, ``MUT`` (short for "Module-Under-Test"). -Running Tests --------------- - -- To run all tests for ``google-cloud-python`` on a single Python version, run - ``py.test`` from your development virtualenv (See - *Using a Development Checkout* above). - -- To run the full set of ``google-cloud-python`` tests on all platforms, install - ``tox`` (https://testrun.org/tox/) into a system Python. The ``tox`` console - script will be installed into the scripts location for that Python. While - ``cd``'ed to the ``google-cloud-python`` checkout root directory (it contains - ``tox.ini``), invoke the ``tox`` console script. This will read the - ``tox.ini`` file and execute the tests on multiple Python versions and - platforms; while it runs, it creates a virtualenv for each version/platform - combination. For example:: - - $ sudo /usr/bin/pip install tox - $ cd ~/hack-on-google-cloud-python/ - $ /usr/bin/tox - +******************** Running System Tests --------------------- +******************** -- To run system tests you can execute:: +- To run system tests for a given package, you can execute:: - $ tox -e system-tests - $ tox -e system-tests3 + $ nox -f datastore/noxfile.py -s system-3.7 + $ nox -f datastore/noxfile.py -s system-2.7 - or run only system tests for a particular package via:: + .. note:: - $ python system_tests/run_system_test.py --package {package} - $ python3 system_tests/run_system_test.py --package {package} - - To run a subset of the system tests:: - - $ tox -e system-tests -- datastore storage - $ python system_tests/attempt_system_tests.py datastore storage + System tests are only configured to run under Python 2.7 and + Python 3.7. For expediency, we do not run them in older versions + of Python 3. This alone will not run the tests. You'll need to change some local auth settings and change some configuration in your project to @@ -161,20 +169,20 @@ Running System Tests authentication to your project: - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; - see ``system_tests/app_credentials.json.sample`` as an example. Such a file - can be downloaded directly from the developer's console by clicking + Such a file can be downloaded directly from the developer's console by clicking "Generate new JSON key". See private key `docs `__ - for more details. In order for Logging system tests to work, the Service Account - will also have to be made a project Owner. This can be changed under "IAM & Admin". - - ``GOOGLE_CLOUD_TESTS_API_KEY``: The API key for your project with - the Google Translate API (and others) enabled. + for more details. -- Examples of these can be found in ``system_tests/local_test_setup.sample``. We - recommend copying this to ``system_tests/local_test_setup``, editing the - values and sourcing them into your environment:: + - In order for Logging system tests to work, the Service Account + will also have to be made a project ``Owner``. This can be changed under + "IAM & Admin". Additionally, ``cloud-logs@google.com`` must be given + ``Editor`` permissions on the project. - $ source system_tests/local_test_setup +- Once you have downloaded your json keys, set the environment variable + ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file:: + + $ export GOOGLE_APPLICATION_CREDENTIALS="/Users//path/to/app_credentials.json" - For datastore tests, you'll need to create composite `indexes `__ @@ -182,11 +190,7 @@ Running System Tests `tool `__:: # Install the app (App Engine Command Line Interface) component. - $ gcloud components update app - - # See https://cloud.google.com/sdk/crypto for details on PyOpenSSL and - # http://stackoverflow.com/a/25067729/1068170 for why we must persist. - $ export CLOUDSDK_PYTHON_SITEPACKAGES=1 + $ gcloud components install app-engine-python # Authenticate the gcloud tool with your account. $ GOOGLE_APPLICATION_CREDENTIALS="path/to/app_credentials.json" @@ -194,132 +198,66 @@ Running System Tests > --key-file=${GOOGLE_APPLICATION_CREDENTIALS} # Create the indexes - $ gcloud preview datastore create-indexes system_tests/data/index.yaml - - # Restore your environment to its previous state. - $ unset CLOUDSDK_PYTHON_SITEPACKAGES + $ gcloud datastore indexes create datastore/tests/system/index.yaml - For datastore query tests, you'll need stored data in your dataset. To populate this data, run:: - $ python system_tests/populate_datastore.py + $ python datastore/tests/system/utils/populate_datastore.py - If you make a mistake during development (i.e. a failing test that prevents clean-up) you can clear all system test data from your datastore instance via:: - $ python system_tests/clear_datastore.py - -- System tests can also be run against local `emulators`_ that mock - the production services. To run the system tests with the - ``datastore`` emulator:: - - $ tox -e datastore-emulator - $ GOOGLE_CLOUD_DISABLE_GRPC=true tox -e datastore-emulator - - This also requires that the ``gcloud`` command line tool is - installed. If you'd like to run them directly (outside of a - ``tox`` environment), first start the emulator and - take note of the process ID:: - - $ gcloud beta emulators datastore start --no-legacy 2>&1 > log.txt & - [1] 33333 - - then determine the environment variables needed to interact with - the emulator:: - - $ gcloud beta emulators datastore env-init - export DATASTORE_LOCAL_HOST=localhost:8417 - export DATASTORE_HOST=http://localhost:8417 - export DATASTORE_DATASET=google-cloud-settings-app-id - export DATASTORE_PROJECT_ID=google-cloud-settings-app-id - - using these environment variables run the emulator:: - - $ DATASTORE_HOST=http://localhost:8471 \ - > DATASTORE_DATASET=google-cloud-settings-app-id \ - > GOOGLE_CLOUD_NO_PRINT=true \ - > python system_tests/run_system_test.py \ - > --package=datastore --ignore-requirements + $ python datastore/tests/system/utils/clear_datastore.py - and after completion stop the emulator and any child - processes it spawned:: - - $ kill -- -33333 - -.. _emulators: https://cloud.google.com/sdk/gcloud/reference/beta/emulators/ - -- To run the system tests with the ``pubsub`` emulator:: - - $ tox -e pubsub-emulator - $ GOOGLE_CLOUD_DISABLE_GRPC=true tox -e pubsub-emulator - - If you'd like to run them directly (outside of a ``tox`` environment), first - start the emulator and take note of the process ID:: - - $ gcloud beta emulators pubsub start 2>&1 > log.txt & - [1] 44444 - - then determine the environment variables needed to interact with - the emulator:: - - $ gcloud beta emulators pubsub env-init - export PUBSUB_EMULATOR_HOST=localhost:8897 - - using these environment variables run the emulator:: - - $ PUBSUB_EMULATOR_HOST=localhost:8897 \ - > python system_tests/run_system_test.py \ - > --package=pubsub - - and after completion stop the emulator and any child - processes it spawned:: - - $ kill -- -44444 +************* Test Coverage -------------- +************* - The codebase *must* have 100% test statement coverage after each commit. - You can test coverage via ``tox -e cover``. + You can test coverage via ``nox -s cover``. +****************************************************** Documentation Coverage and Building HTML Documentation ------------------------------------------------------- +****************************************************** If you fix a bug, and the bug requires an API or behavior modification, all documentation in this package which references that API or behavior must be changed to reflect the bug fix, ideally in the same commit that fixes the bug or adds the feature. -To build and review docs (where ``$VENV`` refers to the virtualenv you're +To build and review docs (where ``${VENV}`` refers to the virtualenv you're using to develop ``google-cloud-python``): -1. After following the steps above in "Using a Development Checkout", install +#. After following the steps above in "Using a Development Checkout", install Sphinx and all development requirements in your virtualenv:: - $ cd ~/hack-on-google-cloud-python - $ $VENV/bin/pip install Sphinx + $ cd ${HOME}/hack-on-google-cloud-python + $ ${VENV}/bin/pip install Sphinx -2. Change into the ``docs`` directory within your ``google-cloud-python`` checkout and +#. Change into the ``docs`` directory within your ``google-cloud-python`` checkout and execute the ``make`` command with some flags:: - $ cd ~/hack-on-google-cloud-python/google-cloud-python/docs - $ make clean html SPHINXBUILD=$VENV/bin/sphinx-build + $ cd ${HOME}/hack-on-google-cloud-python/google-cloud-python/docs + $ make clean html SPHINXBUILD=${VENV}/bin/sphinx-build The ``SPHINXBUILD=...`` argument tells Sphinx to use the virtualenv Python, which will have both Sphinx and ``google-cloud-python`` (for API documentation generation) installed. -3. Open the ``docs/_build/html/index.html`` file to see the resulting HTML +#. Open the ``docs/_build/html/index.html`` file to see the resulting HTML rendering. -As an alternative to 1. and 2. above, if you have ``tox`` installed, you +As an alternative to 1. and 2. above, if you have ``nox`` installed, you can build the docs via:: - $ tox -e docs + $ nox -s docs +******************************************** Note About ``README`` as it pertains to PyPI --------------------------------------------- +******************************************** The `description on PyPI`_ for the project comes directly from the ``README``. Due to the reStructuredText (``rst``) parser used by @@ -328,58 +266,51 @@ instead of ``https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst``) may cause problems creating links or rendering the description. -.. _description on PyPI: https://pypi.python.org/pypi/google-cloud - -Travis Configuration and Build Optimizations --------------------------------------------- +.. _description on PyPI: https://pypi.org/project/google-cloud/ -All build scripts in the ``.travis.yml`` configuration file which have -Python dependencies are specified in the ``tox.ini`` configuration. -They are executed in the Travis build via ``tox -e {ENV}`` where -``{ENV}`` is the environment being tested. +********************** +CircleCI Configuration +********************** -If new ``tox`` environments are added to be run in a Travis build, they -should be listed in ``[tox].envlist`` as a default environment. +All build scripts in the ``.circleci/config.yml`` configuration file which have +Python dependencies are specified in the ``nox.py`` configuration. +They are executed in the Travis build via ``nox -s ${ENV}`` where +``${ENV}`` is the environment being tested. -We speed up builds by using the Travis `caching feature`_. - -.. _caching feature: https://docs.travis-ci.com/user/caching/#pip-cache - -We intentionally **do not** cache the ``.tox/`` directory. Instead, we -allow the ``tox`` environments to be re-built for every build. This -way, we'll always get the latest versions of our dependencies and any -caching or wheel optimization to be done will be handled automatically -by ``pip``. +************************* Supported Python Versions -------------------------- +************************* We support: -- `Python 2.7`_ -- `Python 3.4`_ - `Python 3.5`_ +- `Python 3.6`_ +- `Python 3.7`_ -.. _Python 2.7: https://docs.python.org/2.7/ -.. _Python 3.4: https://docs.python.org/3.4/ .. _Python 3.5: https://docs.python.org/3.5/ +.. _Python 3.6: https://docs.python.org/3.6/ +.. _Python 3.7: https://docs.python.org/3.7/ -Supported versions can be found in our ``tox.ini`` `config`_. -.. _config: https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/tox.ini +Supported versions can be found in our ``noxfile.py`` `config`_. + +.. _config: https://github.com/googleapis/google-cloud-python/blob/master/noxfile.py We explicitly decided not to support `Python 2.5`_ due to `decreased usage`_ and lack of continuous integration `support`_. .. _Python 2.5: https://docs.python.org/2.5/ .. _decreased usage: https://caremad.io/2013/10/a-look-at-pypi-downloads/ -.. _support: http://blog.travis-ci.com/2013-11-18-upcoming-build-environment-updates/ +.. _support: https://blog.travis-ci.com/2013-11-18-upcoming-build-environment-updates/ We have `dropped 2.6`_ as a supported version as well since Python 2.6 is no longer supported by the core development team. +Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. + We also explicitly decided to support Python 3 beginning with version -3.4. Reasons for this include: +3.5. Reasons for this include: - Encouraging use of newest versions of Python 3 - Taking the lead of `prominent`_ open-source `projects`_ @@ -389,25 +320,33 @@ We also explicitly decided to support Python 3 beginning with version .. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django .. _projects: http://flask.pocoo.org/docs/0.10/python3/ .. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ -.. _dropped 2.6: https://github.com/GoogleCloudPlatform/google-cloud-python/issues/995 +.. _dropped 2.6: https://github.com/googleapis/google-cloud-python/issues/995 +********** Versioning ----------- +********** This library follows `Semantic Versioning`_. .. _Semantic Versioning: http://semver.org/ -It is currently in major version zero (``0.y.z``), which means that anything -may change at any time and the public API should not be considered +Some packages are currently in major version zero (``0.y.z``), which means that +anything may change at any time and the public API should not be considered stable. +****************************** Contributor License Agreements ------------------------------- +****************************** -Before we can accept your pull requests you'll need to sign a Contributor License Agreement (CLA): +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): -- **If you are an individual writing original source code** and **you own the intellectual property**, then you'll need to sign an `individual CLA `__. -- **If you work for a company that wants to allow you to contribute your work**, then you'll need to sign a `corporate CLA `__. +- **If you are an individual writing original source code** and **you own the + intellectual property**, then you'll need to sign an + `individual CLA `__. +- **If you work for a company that wants to allow you to contribute your work**, + then you'll need to sign a + `corporate CLA `__. -You can sign these electronically (just scroll to the bottom). After that, we'll be able to accept your pull requests. +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ae20b86a84fd..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.rst -graft google -global-exclude *.pyc diff --git a/Makefile.bigtable_v2 b/Makefile.bigtable_v2 index 46a958cf4234..b739053e0116 100644 --- a/Makefile.bigtable_v2 +++ b/Makefile.bigtable_v2 @@ -1,15 +1,15 @@ GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv GENERATED_DIR=$(shell pwd)/generated_python GENERATED_SUBDIR=_generated -BIGTABLE_DIR=$(shell pwd)/google/cloud/bigtable/$(GENERATED_SUBDIR) +BIGTABLE_DIR=$(shell pwd)/bigtable/google/cloud/bigtable/$(GENERATED_SUBDIR) PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb help: - @echo 'Makefile for google-cloud-python Bigtable protos ' - @echo ' ' - @echo ' make generate Generates the protobuf modules ' - @echo ' make clean Clean generated files ' + @echo 'Makefile for google-cloud-python Bigtable protos ' + @echo ' ' + @echo ' make generate Generates the protobuf modules' + @echo ' make clean Clean generated files ' generate: # Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools @@ -44,7 +44,6 @@ generate: # Copy over the *.proto files into our library. cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR) cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR) - cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR) # Rename all *.proto files in our library with an # underscore and remove executable bit. cd $(BIGTABLE_DIR) && \ @@ -52,12 +51,7 @@ generate: chmod -x $$filename ; \ mv $$filename _$$filename ; \ done - # Separate the gRPC parts of the operations service from the - # non-gRPC parts so that the protos from `googleapis-common-protos` - # can be used without gRPC. GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \ - GENERATED_SUBDIR=$(GENERATED_SUBDIR) \ - python scripts/make_operations_grpc.py # Rewrite the imports in the generated *_pb2.py files. python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py diff --git a/Makefile.datastore b/Makefile.datastore deleted file mode 100644 index 837f5ba26390..000000000000 --- a/Makefile.datastore +++ /dev/null @@ -1,53 +0,0 @@ -GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv -GENERATED_DIR=$(shell pwd)/generated_python -DATASTORE_DIR=$(shell pwd)/google/cloud/datastore/_generated -PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc -GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb - -help: - @echo 'Makefile for google-cloud-python Bigtable protos ' - @echo ' ' - @echo ' make generate Generates the protobuf modules ' - @echo ' make clean Clean generated files ' - -generate: - # Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools - [ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV) - $(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools - # Retrieve git repos that have our *.proto files. - [ -d googleapis-pb ] || git clone https://github.com/googleapis/googleapis googleapis-pb --depth=1 - cd googleapis-pb && git pull origin master - # Make the directory where our *_pb2.py files will go. - mkdir -p $(GENERATED_DIR) - # Generate all *_pb2.py files that do not require gRPC. - $(PROTOC_CMD) \ - --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \ - --python_out=$(GENERATED_DIR) \ - $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1/datastore.proto \ - $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1/entity.proto \ - $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1/query.proto - # Move the newly generated *_pb2.py files into our library. - cp $(GENERATED_DIR)/google/datastore/v1/* $(DATASTORE_DIR) - # Remove all existing *.proto files before we replace - rm -f $(DATASTORE_DIR)/*.proto - # Copy over the *.proto files into our library. - cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1/*.proto $(DATASTORE_DIR) - # Rename all *.proto files in our library with an - # underscore and remove executable bit. - cd $(DATASTORE_DIR) && \ - for filename in *.proto; do \ - chmod -x $$filename ; \ - mv $$filename _$$filename ; \ - done - # Separate the gRPC parts of the datastore service from the - # non-gRPC parts so that the protos can be used without gRPC. - GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \ - GENERATED_SUBDIR=$(GENERATED_SUBDIR) \ - python scripts/make_datastore_grpc.py - # Rewrite the imports in the generated *_pb2.py files. - python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py - -clean: - rm -fr $(GENERATED_DIR) - -.PHONY: generate clean diff --git a/README.rst b/README.rst index 39568ec8a5a8..f0820498cc21 100644 --- a/README.rst +++ b/README.rst @@ -1,47 +1,170 @@ Google Cloud Python Client ========================== - Python idiomatic client for `Google Cloud Platform`_ services. +Python idiomatic clients for `Google Cloud Platform`_ services. .. _Google Cloud Platform: https://cloud.google.com/ -|pypi| |build| |appveyor| |coverage| |versions| +**Heads up**! These libraries are supported on App Engine standard's `Python 3 runtime`_ but are *not* supported on App Engine's `Python 2 runtime`_. -- `Homepage`_ -- `API Documentation`_ +.. _Python 3 runtime: https://cloud.google.com/appengine/docs/standard/python3 +.. _Python 2 runtime: https://cloud.google.com/appengine/docs/standard/python -.. _Homepage: https://googlecloudplatform.github.io/google-cloud-python/ -.. _API Documentation: http://googlecloudplatform.github.io/google-cloud-python/ - -This client supports the following Google Cloud Platform services: +General Availability +-------------------- -- `Google Cloud Datastore`_ -- `Google Cloud Storage`_ -- `Google Cloud Pub/Sub`_ -- `Google BigQuery`_ -- `Google Cloud Resource Manager`_ -- `Google Stackdriver Logging`_ -- `Google Stackdriver Monitoring`_ +**GA** (general availability) indicates that the client library for a +particular service is stable, and that the code surface will not change in +backwards-incompatible ways unless either absolutely necessary (e.g. because +of critical security issues) or with an extensive deprecation period. +Issues and requests against GA libraries are addressed with the highest +priority. + +.. note:: + + Sub-components of GA libraries explicitly marked as beta in the + import path (e.g. ``google.cloud.language_v1beta2``) should be considered + to be beta. + +The following client libraries have **GA** support: + +- `Google BigQuery`_ (`BigQuery README`_) +- `Google Cloud Datastore`_ (`Datastore README`_) +- `Google Cloud KMS`_ (`KMS README`_) +- `Google Cloud Natural Language`_ (`Natural Language README`_) +- `Google Cloud Scheduler`_ (`Scheduler README`_) +- `Google Cloud Spanner`_ (`Spanner README`_) +- `Google Cloud Speech`_ (`Speech README`_) +- `Google Cloud Storage`_ (`Storage README`_) +- `Google Cloud Tasks`_ (`Tasks README`_) +- `Google Cloud Translation`_ (`Translation README`_) +- `Stackdriver Logging`_ (`Logging README`_) + +.. _Google BigQuery: https://pypi.org/project/google-cloud-bigquery/ +.. _BigQuery README: https://github.com/googleapis/google-cloud-python/tree/master/bigquery +.. _Google Cloud Datastore: https://pypi.org/project/google-cloud-datastore/ +.. _Datastore README: https://github.com/googleapis/google-cloud-python/tree/master/datastore +.. _Google Cloud KMS: https://pypi.org/project/google-cloud-kms/ +.. _KMS README: https://github.com/googleapis/google-cloud-python/tree/master/kms +.. _Google Cloud Natural Language: https://pypi.org/project/google-cloud-language/ +.. _Natural Language README: https://github.com/googleapis/google-cloud-python/tree/master/language +.. _Google Cloud Spanner: https://pypi.org/project/google-cloud-spanner +.. _Spanner README: https://github.com/googleapis/google-cloud-python/tree/master/spanner +.. _Google Cloud Speech: https://pypi.org/project/google-cloud-speech/ +.. _Speech README: https://github.com/googleapis/google-cloud-python/tree/master/speech +.. _Google Cloud Storage: https://pypi.org/project/google-cloud-storage/ +.. _Storage README: https://github.com/googleapis/google-cloud-python/tree/master/storage +.. _Google Cloud Tasks: https://pypi.org/project/google-cloud-tasks/ +.. _Tasks README: https://github.com/googleapis/google-cloud-python/tree/master/tasks +.. _Google Cloud Translation: https://pypi.org/project/google-cloud-translate/ +.. _Translation README: https://github.com/googleapis/google-cloud-python/tree/master/translate +.. _Google Cloud Scheduler: https://pypi.org/project/google-cloud-scheduler/ +.. _Scheduler README: https://github.com/googleapis/google-cloud-python/tree/master/scheduler +.. _Stackdriver Logging: https://pypi.org/project/google-cloud-logging/ +.. _Logging README: https://github.com/googleapis/google-cloud-python/tree/master/logging + +Beta Support +------------ -.. _Google Cloud Datastore: https://github.com/GoogleCloudPlatform/google-cloud-python#google-cloud-datastore -.. _Google Cloud Storage: https://github.com/GoogleCloudPlatform/google-cloud-python#google-cloud-storage -.. _Google Cloud Pub/Sub: https://github.com/GoogleCloudPlatform/google-cloud-python#google-cloud-pubsub -.. _Google BigQuery: https://github.com/GoogleCloudPlatform/google-cloud-python#google-bigquery -.. _Google Cloud Resource Manager: https://github.com/GoogleCloudPlatform/google-cloud-python#google-cloud-resource-manager -.. _Google Stackdriver Logging: https://github.com/GoogleCloudPlatform/google-cloud-python#google-stackdriver-logging -.. _Google Stackdriver Monitoring: https://github.com/GoogleCloudPlatform/google-cloud-python#google-stackdriver-monitoring +**Beta** indicates that the client library for a particular service is +mostly stable and is being prepared for release. Issues and requests +against beta libraries are addressed with a higher priority. + +The following client libraries have **beta** support: + +- `Google Cloud Bigtable`_ (`Bigtable README`_) +- `Google Cloud Firestore`_ (`Firestore README`_) +- `Google Cloud Pub/Sub`_ (`Pub/Sub README`_) +- `Google Cloud Video Intelligence`_ (`Video Intelligence README`_) +- `Google Cloud Vision`_ (`Vision README`_) + +.. _Google Cloud Bigtable: https://pypi.org/project/google-cloud-bigtable/ +.. _Bigtable README: https://github.com/googleapis/google-cloud-python/tree/master/bigtable +.. _Google Cloud Firestore: https://pypi.org/project/google-cloud-firestore/ +.. _Firestore README: https://github.com/googleapis/google-cloud-python/tree/master/firestore +.. _Google Cloud Pub/Sub: https://pypi.org/project/google-cloud-pubsub/ +.. _Pub/Sub README: https://github.com/googleapis/google-cloud-python/tree/master/pubsub +.. _Google Cloud Video Intelligence: https://pypi.org/project/google-cloud-videointelligence +.. _Video Intelligence README: https://github.com/googleapis/google-cloud-python/tree/master/videointelligence +.. _Google Cloud Vision: https://pypi.org/project/google-cloud-vision/ +.. _Vision README: https://github.com/googleapis/google-cloud-python/tree/master/vision + + +Alpha Support +------------- + +**Alpha** indicates that the client library for a particular service is +still a work-in-progress and is more likely to get backwards-incompatible +updates. See `versioning`_ for more details. + +The following client libraries have **alpha** support: + +- `Google Cloud Asset`_ (`Asset README`_) +- `Google Cloud AutoML`_ (`AutoML README`_) +- `Google BigQuery Data Transfer`_ (`BigQuery Data Transfer README`_) +- `Google Cloud Bigtable - HappyBase`_ (`HappyBase README`_) +- `Google Cloud Container`_ (`Container README`_) +- `Google Cloud Container Analysis`_ (`Container Analysis README`_) +- `Google Cloud Dataproc`_ (`Dataproc README`_) +- `Google Cloud DLP`_ (`DLP README`_) +- `Google Cloud DNS`_ (`DNS README`_) +- `Google Cloud IoT`_ (`IoT README`_) +- `Google Cloud Memorystore for Redis`_ (`Redis README`_) +- `Google Cloud Resource Manager`_ (`Resource Manager README`_) +- `Google Cloud Runtime Configuration`_ (`Runtime Config README`_) +- `Google Cloud Security Scanner`_ (`Security Scanner README`_ ) +- `Google Cloud Trace`_ (`Trace README`_) +- `Google Cloud Text-to-Speech`_ (`Text-to-Speech README`_) +- `Grafeas`_ (`Grafeas README`_) +- `Stackdriver Error Reporting`_ (`Error Reporting README`_) +- `Stackdriver Monitoring`_ (`Monitoring README`_) + +.. _Google Cloud Asset: https://pypi.org/project/google-cloud-asset/ +.. _Asset README: https://github.com/googleapis/google-cloud-python/blob/master/asset +.. _Google Cloud AutoML: https://pypi.org/project/google-cloud-automl/ +.. _AutoML README: https://github.com/googleapis/google-cloud-python/blob/master/automl +.. _Google BigQuery Data Transfer: https://pypi.org/project/google-cloud-bigquery-datatransfer/ +.. _BigQuery Data Transfer README: https://github.com/googleapis/google-cloud-python/tree/master/bigquery_datatransfer +.. _Google Cloud Bigtable - HappyBase: https://pypi.org/project/google-cloud-happybase/ +.. _HappyBase README: https://github.com/googleapis/google-cloud-python-happybase +.. _Google Cloud Container: https://pypi.org/project/google-cloud-container/ +.. _Container README: https://github.com/googleapis/google-cloud-python/tree/master/container +.. _Google Cloud Container Analysis: https://pypi.org/project/google-cloud-containeranalysis/ +.. _Container Analysis README: https://github.com/googleapis/google-cloud-python/tree/master/containeranalysis +.. _Google Cloud Dataproc: https://pypi.org/project/google-cloud-dataproc/ +.. _Dataproc README: https://github.com/googleapis/google-cloud-python/tree/master/dataproc +.. _Google Cloud DLP: https://pypi.org/project/google-cloud-dlp/ +.. _DLP README: https://github.com/googleapis/google-cloud-python/tree/master/dlp +.. _Google Cloud DNS: https://pypi.org/project/google-cloud-dns/ +.. _DNS README: https://github.com/googleapis/google-cloud-python/tree/master/dns +.. _Google Cloud IoT: https://pypi.org/project/google-cloud-iot/ +.. _IoT README: https://github.com/googleapis/google-cloud-python/tree/master/iot +.. _Google Cloud Memorystore for Redis: https://pypi.org/project/google-cloud-redis/ +.. _Redis README: https://github.com/googleapis/google-cloud-python/tree/master/redis +.. _Google Cloud Resource Manager: https://pypi.org/project/google-cloud-resource-manager/ +.. _Resource Manager README: https://github.com/googleapis/google-cloud-python/tree/master/resource_manager +.. _Google Cloud Runtime Configuration: https://pypi.org/project/google-cloud-runtimeconfig/ +.. _Runtime Config README: https://github.com/googleapis/google-cloud-python/tree/master/runtimeconfig +.. _Google Cloud Security Scanner: https://pypi.org/project/google-cloud-websecurityscanner/ +.. _Security Scanner README: https://github.com/googleapis/google-cloud-python/blob/master/websecurityscanner +.. _Google Cloud Text-to-Speech: https://pypi.org/project/google-cloud-texttospeech/ +.. _Text-to-Speech README: https://github.com/googleapis/google-cloud-python/tree/master/texttospeech +.. _Google Cloud Trace: https://pypi.org/project/google-cloud-trace/ +.. _Trace README: https://github.com/googleapis/google-cloud-python/tree/master/trace +.. _Grafeas: https://pypi.org/project/grafeas/ +.. _Grafeas README: https://github.com/googleapis/google-cloud-python/tree/master/grafeas +.. _Stackdriver Error Reporting: https://pypi.org/project/google-cloud-error-reporting/ +.. _Error Reporting README: https://github.com/googleapis/google-cloud-python/tree/master/error_reporting +.. _Stackdriver Monitoring: https://pypi.org/project/google-cloud-monitoring/ +.. _Monitoring README: https://github.com/googleapis/google-cloud-python/tree/master/monitoring + +.. _versioning: https://github.com/googleapis/google-cloud-python/blob/master/CONTRIBUTING.rst#versioning If you need support for other Google APIs, check out the `Google APIs Python Client library`_. .. _Google APIs Python Client library: https://github.com/google/google-api-python-client -Quick Start ------------ - -:: - - $ pip install --upgrade google-cloud Example Applications -------------------- @@ -53,6 +176,7 @@ Example Applications .. _tutorial: https://cloud.google.com/python .. _google-cloud-python-expenses-demo: https://github.com/GoogleCloudPlatform/google-cloud-python-expenses-demo + Authentication -------------- @@ -61,276 +185,32 @@ Check out the `Authentication section`_ in our documentation to learn more. You may also find the `authentication document`_ shared by all the ``google-cloud-*`` libraries to be helpful. -.. _Authentication section: http://google-cloud-python.readthedocs.io/en/latest/google-cloud-auth.html -.. _authentication document: https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/authentication - -Google Cloud Datastore ----------------------- - -Google `Cloud Datastore`_ (`Datastore API docs`_) is a fully managed, schemaless -database for storing non-relational data. Cloud Datastore automatically scales -with your users and supports ACID transactions, high availability of reads and -writes, strong consistency for reads and ancestor queries, and eventual -consistency for all other queries. - -.. _Cloud Datastore: https://cloud.google.com/datastore/docs -.. _Datastore API docs: https://cloud.google.com/datastore/docs/ - -See the ``google-cloud-python`` API `datastore documentation`_ to learn how to -interact with the Cloud Datastore using this Client Library. - -.. _datastore documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/datastore-client.html - -See the `official Google Cloud Datastore documentation`_ for more details on how -to activate Cloud Datastore for your project. - -.. _official Google Cloud Datastore documentation: https://cloud.google.com/datastore/docs/activate - -.. code:: python - - from google.cloud import datastore - # Create, populate and persist an entity - entity = datastore.Entity(key=datastore.Key('EntityKind')) - entity.update({ - 'foo': u'bar', - 'baz': 1337, - 'qux': False, - }) - # Then query for entities - query = datastore.Query(kind='EntityKind') - for result in query.fetch(): - print result - -Google Cloud Storage --------------------- - -Google `Cloud Storage`_ (`Storage API docs`_) allows you to store data on Google -infrastructure with very high reliability, performance and availability, and can -be used to distribute large data objects to users via direct download. - -.. _Cloud Storage: https://cloud.google.com/storage/docs -.. _Storage API docs: https://cloud.google.com/storage/docs/json_api/v1 - -See the ``google-cloud-python`` API `storage documentation`_ to learn how to connect -to Cloud Storage using this Client Library. - -.. _storage documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/storage-client.html - -You need to create a Google Cloud Storage bucket to use this client library. -Follow along with the `official Google Cloud Storage documentation`_ to learn -how to create a bucket. - -.. _official Google Cloud Storage documentation: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets - -.. code:: python - - from google.cloud import storage - client = storage.Client() - bucket = client.get_bucket('bucket-id-here') - # Then do other things... - blob = bucket.get_blob('remote/path/to/file.txt') - print blob.download_as_string() - blob.upload_from_string('New contents!') - blob2 = bucket.blob('remote/path/storage.txt') - blob2.upload_from_filename(filename='/local/path.txt') - -Google Cloud Pub/Sub --------------------- - -Google `Cloud Pub/Sub`_ (`Pub/Sub API docs`_) is designed to provide reliable, -many-to-many, asynchronous messaging between applications. Publisher -applications can send messages to a ``topic`` and other applications can -subscribe to that topic to receive the messages. By decoupling senders and -receivers, Google Cloud Pub/Sub allows developers to communicate between -independently written applications. - -.. _Cloud Pub/Sub: https://cloud.google.com/pubsub/docs -.. _Pub/Sub API docs: https://cloud.google.com/pubsub/reference/rest/ - -See the ``google-cloud-python`` API `Pub/Sub documentation`_ to learn how to connect -to Cloud Pub/Sub using this Client Library. - -.. _Pub/Sub documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/pubsub-usage.html - -To get started with this API, you'll need to create - -.. code:: python - - from google.cloud import pubsub - - client = pubsub.Client() - topic = client.topic('topic_name') - topic.create() - - topic.publish('this is the message_payload', - attr1='value1', attr2='value2') - -Google BigQuery ---------------- - -Querying massive datasets can be time consuming and expensive without the -right hardware and infrastructure. Google `BigQuery`_ (`BigQuery API docs`_) -solves this problem by enabling super-fast, SQL-like queries against -append-only tables, using the processing power of Google's infrastructure. - -.. _BigQuery: https://cloud.google.com/bigquery/what-is-bigquery -.. _BigQuery API docs: https://cloud.google.com/bigquery/docs/reference/v2/ - -This package is still being implemented, but it is almost complete! - -Load data from CSV -~~~~~~~~~~~~~~~~~~ - -.. code:: python +.. _Authentication section: https://google-cloud-python.readthedocs.io/en/latest/core/auth.html +.. _authentication document: https://github.com/googleapis/google-cloud-common/tree/master/authentication - import csv - - from google.cloud import bigquery - from google.cloud.bigquery import SchemaField - - client = bigquery.Client() - - dataset = client.dataset('dataset_name') - dataset.create() # API request - - SCHEMA = [ - SchemaField('full_name', 'STRING', mode='required'), - SchemaField('age', 'INTEGER', mode='required'), - ] - table = dataset.table('table_name', SCHEMA) - table.create() - - with open('csv_file', 'rb') as readable: - table.upload_from_file( - readable, source_format='CSV', skip_leading_rows=1) - -Perform a synchronous query -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: python - - # Perform a synchronous query. - QUERY = ( - 'SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] ' - 'WHERE state = "TX"') - query = client.run_sync_query('%s LIMIT 100' % QUERY) - query.timeout_ms = TIMEOUT_MS - query.run() - - for row in query.rows: - print row - - -See the ``google-cloud-python`` API `BigQuery documentation`_ to learn how to connect -to BigQuery using this Client Library. - -.. _BigQuery documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/bigquery-usage.html - -Google Cloud Resource Manager ------------------------------ - -The Cloud `Resource Manager`_ API (`Resource Manager API docs`_) provides -methods that you can use to programmatically manage your projects in the -Google Cloud Platform. - -.. _Resource Manager: https://cloud.google.com/resource-manager/ -.. _Resource Manager API docs: https://cloud.google.com/resource-manager/reference/rest/ - -See the ``google-cloud-python`` API `Resource Manager documentation`_ to learn how to -manage projects using this Client Library. - -.. _Resource Manager documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/resource-manager-api.html - -Google Stackdriver Logging --------------------------- - -`Stackdriver Logging`_ API (`Logging API docs`_) allows you to store, search, -analyze, monitor, and alert on log data and events from Google Cloud Platform. - -.. _Stackdriver Logging: https://cloud.google.com/logging/ -.. _Logging API docs: https://cloud.google.com/logging/docs/ - -.. code:: python - - from google.cloud import logging - client = logging.Client() - logger = client.logger('log_name') - logger.log_text("A simple entry") # API call - -Example of fetching entries: - -.. code:: python - - entries, token = logger.list_entries() - for entry in entries: - print entry.payload - -See the ``google-cloud-python`` API `logging documentation`_ to learn how to connect -to Stackdriver Logging using this Client Library. - -.. _logging documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/logging-usage.html - -Google Stackdriver Monitoring ------------------------------ - -`Stackdriver Monitoring`_ (`Monitoring API docs`_) collects metrics, -events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), -hosted uptime probes, application instrumentation, and a variety of common -application components including Cassandra, Nginx, Apache Web Server, -Elasticsearch and many others. Stackdriver ingests that data and generates -insights via dashboards, charts, and alerts. - -This package currently supports all Monitoring API operations other than -writing custom metrics. - -.. _Stackdriver Monitoring: https://cloud.google.com/monitoring/ -.. _Monitoring API docs: https://cloud.google.com/monitoring/api/ref_v3/rest/ - -List available metric types: - -.. code:: python - - from google.cloud import monitoring - client = monitoring.Client() - for descriptor in client.list_metric_descriptors(): - print(descriptor.type) - -Display CPU utilization across your GCE instances during the last five minutes: +Contributing +------------ -.. code:: python +Contributions to this library are always welcome and highly encouraged. - metric = 'compute.googleapis.com/instance/cpu/utilization' - query = client.query(metric, minutes=5) - print(query.as_dataframe()) +See the `CONTRIBUTING doc`_ for more information on how to get started. -See the ``google-cloud-python`` API `monitoring documentation`_ to learn how to connect -to Stackdriver Monitoring using this Client Library. +.. _CONTRIBUTING doc: https://github.com/googleapis/google-cloud-python/blob/master/CONTRIBUTING.rst -.. _monitoring documentation: https://googlecloudplatform.github.io/google-cloud-python/stable/monitoring-usage.html -Contributing ------------- +Community +--------- -Contributions to this library are always welcome and highly encouraged. +Google Cloud Platform Python developers hang out in `Slack`_ in the ``#python`` +channel, click here to `get an invitation`_. -See `CONTRIBUTING`_ for more information on how to get started. +.. _Slack: https://googlecloud-community.slack.com +.. _get an invitation: https://gcp-slack.appspot.com/ -.. _CONTRIBUTING: https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst License ------- -Apache 2.0 - See `LICENSE`_ for more information. - -.. _LICENSE: https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/LICENSE +Apache 2.0 - See `the LICENSE`_ for more information. -.. |build| image:: https://travis-ci.org/GoogleCloudPlatform/google-cloud-python.svg?branch=master - :target: https://travis-ci.org/GoogleCloudPlatform/google-cloud-python -.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/github/googlecloudplatform/google-cloud-python?branch=master&svg=true - :target: https://ci.appveyor.com/project/GoogleCloudPlatform/google-cloud-python -.. |coverage| image:: https://coveralls.io/repos/GoogleCloudPlatform/google-cloud-python/badge.png?branch=master - :target: https://coveralls.io/r/GoogleCloudPlatform/google-cloud-python?branch=master -.. |pypi| image:: https://img.shields.io/pypi/v/google-cloud.svg - :target: https://pypi.python.org/pypi/google-cloud -.. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud.svg - :target: https://pypi.python.org/pypi/google-cloud +.. _the LICENSE: https://github.com/googleapis/google-cloud-python/blob/master/LICENSE diff --git a/api_core/.coveragerc b/api_core/.coveragerc new file mode 100644 index 000000000000..d097511c3124 --- /dev/null +++ b/api_core/.coveragerc @@ -0,0 +1,13 @@ +[run] +branch = True + +[report] +fail_under = 100 +show_missing = True +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ + # Ignore abstract methods + raise NotImplementedError diff --git a/api_core/.flake8 b/api_core/.flake8 new file mode 100644 index 000000000000..3da787cdfbf3 --- /dev/null +++ b/api_core/.flake8 @@ -0,0 +1,11 @@ +[flake8] +import-order-style=google +# Note: this forces all google imports to be in the third group. See +# https://github.com/PyCQA/flake8-import-order/issues/111 +application-import-names=google +ignore = E203, E266, E501, W503 +exclude = + __pycache__, + .git, + *.pyc, + conf.py diff --git a/api_core/.repo-metadata.json b/api_core/.repo-metadata.json new file mode 100644 index 000000000000..6a5f3f53359f --- /dev/null +++ b/api_core/.repo-metadata.json @@ -0,0 +1,9 @@ +{ + "name": "google-api-core", + "name_pretty": "Google API client core library", + "client_documentation": "https://googleapis.dev/python/google-api-core/latest", + "release_level": "ga", + "language": "python", + "repo": "googleapis/google-cloud-python", + "distribution_name": "google-api-core" +} \ No newline at end of file diff --git a/api_core/CHANGELOG.md b/api_core/CHANGELOG.md new file mode 100644 index 000000000000..74ab16564dbc --- /dev/null +++ b/api_core/CHANGELOG.md @@ -0,0 +1,349 @@ +# Changelog + +[PyPI History][1] + +[1]: https://pypi.org/project/google-api-core/#history + +## 1.14.2 + +07-30-2019 14:08 PDT + + +### Documentation +- Add client_options documentation. ([#8834](https://github.com/googleapis/google-cloud-python/pull/8834)) + +## 1.14.1 + +07-30-2019 12:24 PDT + + +### Implementation Changes +- Remove error log entry on clean BiDi shutdown. ([#8806](https://github.com/googleapis/google-cloud-python/pull/8806)) +- Forward 'timeout' arg from 'exception' to `_blocking_poll`. ([#8735](https://github.com/googleapis/google-cloud-python/pull/8735)) + +### Documentation +- Update intersphinx mapping for requests. ([#8805](https://github.com/googleapis/google-cloud-python/pull/8805)) +- Link to googleapis.dev documentation in READMEs. ([#8705](https://github.com/googleapis/google-cloud-python/pull/8705)) + +## 1.14.0 + +07-17-2019 13:16 PDT + + +### New Features +- Firestore: Add `should_terminate` predicate for clean BiDi shutdown. ([#8650](https://github.com/googleapis/google-cloud-python/pull/8650)) + +### Dependencies +- Update pins of 'googleapis-common-protos. ([#8688](https://github.com/googleapis/google-cloud-python/pull/8688)) + +### Documentation +- Add compatibility check badges to READMEs. ([#8288](https://github.com/googleapis/google-cloud-python/pull/8288)) + +### Internal / Testing Changes +- All: Add docs job to publish to googleapis.dev. ([#8464](https://github.com/googleapis/google-cloud-python/pull/8464)) + +## 1.13.0 + +06-24-2019 10:34 PDT + +### New Features +- Add `client_options.ClientOptions` object. ([#8265](https://github.com/googleapis/google-cloud-python/pull/8265)) + +## 1.12.0 + +06-18-2019 12:37 PDT + + +### New Features +- Add Throttling to Bidi Reopening. Mitigates ResumableBidiRpc consuming 100% CPU ([#8193](https://github.com/googleapis/google-cloud-python/pull/8193)) + +## 1.11.1 + +05-28-2019 11:19 PDT + + +### Implementation Changes +- Classify 503 Service Unavailable errors as transient. ([#8182](https://github.com/googleapis/google-cloud-python/pull/8182)) + +### Dependencies +- Pin `grpcio < 2.0dev`. ([#8182](https://github.com/googleapis/google-cloud-python/pull/8182)) + +### Internal / Testing Changes +- Add parameterized test for `from_rfc3339` with nanos ([#7675](https://github.com/googleapis/google-cloud-python/pull/7675)) +- Unbreak pytype by silencing a false positive. ([#8106](https://github.com/googleapis/google-cloud-python/pull/8106)) + +## 1.11.0 + +05-15-2019 10:29 PDT + +### New Features + +- Refactor 'client_info' support. ([#7849](https://github.com/googleapis/google-cloud-python/pull/7849)) + +## 1.10.0 + +04-29-2019 10:12 PDT + +### Implementation Changes + +- Append leading zeros for nanosecond precision DateTimes + ([#7663](https://github.com/googleapis/google-cloud-python/pull/7663)) + +### New Features + +- Add `user_agent` property to `ClientInfo` + ([#7799](https://github.com/googleapis/google-cloud-python/pull/7799)) + +## 1.9.0 + +04-05-2019 10:38 PDT + + +### Implementation Changes +- Allow passing metadata as part of creating a bidi ([#7514](https://github.com/googleapis/google-cloud-python/pull/7514)) + +### Internal / Testing Changes +- Update setup.py +- API Core: specify a pytype output directory in setup.cfg. ([#7639](https://github.com/googleapis/google-cloud-python/pull/7639)) + +## 1.8.2 + +03-22-2019 16:27 PDT + + +### Implementation Changes +- Remove classifier for Python 3.4 for end-of-life. ([#7535](https://github.com/googleapis/google-cloud-python/pull/7535)) + +### Internal / Testing Changes +- When re-opening a `ResumableBidiRPC` set `_request_queue_generator` to `None`. ([#7548](https://github.com/googleapis/google-cloud-python/pull/7548)) + +## 1.8.1 + +03-12-2019 12:45 PDT + +### Implementation Changes +- Protect the creation of a background thread in BackgroundConsumer and wait on it starting. ([#7499](https://github.com/googleapis/google-cloud-python/pull/7499)) + +## 1.8.0 + +02-23-2019 15:46 PST + + +### New Features +- Add support to unwrap Anys into wrapped pb2 objects. ([#7430](https://github.com/googleapis/google-cloud-python/pull/7430)) +- Add `Operation.deserialize`. ([#7427](https://github.com/googleapis/google-cloud-python/pull/7427)) + +### Documentation +- Updated client library documentation URLs. ([#7307](https://github.com/googleapis/google-cloud-python/pull/7307)) + +### Internal / Testing Changes +- Fix new lint failure. ([#7382](https://github.com/googleapis/google-cloud-python/pull/7382)) + +## 1.7.0 + +12-17-2018 13:56 PST + +### New Features +- Support converting `DatetimeWithNanos` to / from `google.protobuf.timestamp_pb2.Timestamp`. ([#6919](https://github.com/googleapis/google-cloud-python/pull/6919)) + +### Documentation +- Document Python 2 deprecation. ([#6910](https://github.com/googleapis/google-cloud-python/pull/6910)) +- Add usage example for `google.api_core.iam.Polcy`. ([#6855](https://github.com/googleapis/google-cloud-python/pull/6855)) + +### Internal / Testing Changes +- Work around pytype big for `ABCMeta.register`. ([#6873](https://github.com/googleapis/google-cloud-python/pull/6873)) + +## 1.6.0 + +11-30-2018 12:45 PST + + +### Implementation Changes +- Import stdlib ABCs from 'collections.abc' rather than 'collections'. ([#6451](https://github.com/googleapis/google-cloud-python/pull/6451)) + +### New Features +- Move google.cloud.iam (core) to google.api_core.iam ([#6740](https://github.com/googleapis/google-cloud-python/pull/6740)) +- Add bidi support to api_core. ([#6191](https://github.com/googleapis/google-cloud-python/pull/6191)) + +### Documentation +- Fix typo ([#6532](https://github.com/googleapis/google-cloud-python/pull/6532)) + +### Internal / Testing Changes +- blacken api_core and core ([#6668](https://github.com/googleapis/google-cloud-python/pull/6668)) + +## 1.5.2 + +11-09-2018 14:22 PST + + +### Implementation Changes +- Retry transient errors in 'PollingFuture.result'. ([#6305](https://github.com/googleapis/google-cloud-python/pull/6305)) + +### Dependencies +- Remove hyphen from named extra in api_core. ([#6468](https://github.com/googleapis/google-cloud-python/pull/6468)) +- Bump minimum 'api_core' version for all GAPIC libs to 1.4.1. ([#6391](https://github.com/googleapis/google-cloud-python/pull/6391)) +- Avoid broken 'google-common-apis 1.5.4' release. ([#6355](https://github.com/googleapis/google-cloud-python/pull/6355)) + +## 1.5.1 + +10-29-2018 13:29 PDT + +### Implementation Changes +- Don't URL-encode slashes in gRPC request headers. ([#6310](https://github.com/googleapis/google-cloud-python/pull/6310)) + +### Internal / Testing Changes +- Back out changes from [#6267](https://github.com/googleapis/google-cloud-python/pull/6267) / `api_core-1.6.0a1` release. ([#6328](https://github.com/googleapis/google-cloud-python/pull/6328)) + +## 1.5.0 + +### New Features +- Add bidi, Bidirection Streaming, to api-core ([#6211](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/6211)) + +### Internal / Testing Changes +- Use new Nox ([#6175](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/6175)) + +## 1.4.1 + +### Dependencies +- Pin minimum protobuf dependency to 3.4.0. ([#6132](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/6132)) + +### Internal / Testing Changes +- Add type-checking via pytype to api_core. ([#6116](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/6116)) + +## 1.4.0 + +### Dependencies + +- Add support for gRPC connection management (available when using optional grpc_gcp dependency) ([#5553](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/5553)) ([#5904](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/5904)) +- Update classifiers to drop Python 3.4 and add Python 3.7 ([#5702](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/5702)) + +## 1.3.0 + +### New Features + +- Add protobuf_helpers.field_mask to calculate a field mask from two messages (#5320) + +## 1.2.1 + +### Implementation Changes +- Make client_info work without gRPC installed. (#5075) +- Rename `x-goog-header-params` to `x-goog-request-params` (#5495) + +## 1.2.0 + +### Implementation Changes +- Add close method to grpc Channel (#5333) + +### Internal / Testing Changes +- Fix tests after grpcio update (#5333) +- Add Test runs for Python 3.7 and remove 3.4 (#5295) + +## 1.1.2 + +### Packaging +- Update setuptools before packaging (#5265) + +## 1.1.1 + +### Internal / Testing Changes +- Use `install_requires` for platform dependencies instead of `extras_require` (#4991) +- Update trove classifer to '5 - Production/Stable' + +## 1.1.0 + +### Interface additions + +- Add `datetime_helpers.DatetimeWithNanoSeconds` (#4979) + +### Implementation changes + +- Use a class to wrap grpc streaming errors instead of monkey-patching (#4995) + +## 1.0.0 + +This is the stable v1.0.0 release of google-api-core for Python. Releases after +this will not contain breaking changes. + +### Interface changes and additions + +- Made `api_core.page_iterator.PageIterator.item_to_value` public +- Added ability to specify retry for `Operation` and `polling.Future`. (#4922) + +## 0.1.4 + +### New Features + +- Add `ChannelStub` to `grpc_helpers` for testing gRPC-based clients. (#4705) + +### Notable Implementation Changes + +- Fix handling of gapic metadata when specified as `None`. (#4701) + +## 0.1.3 + +### Notable Implementation Changes + +- Apply scopes to explicitly provided credentials if needed (#4594). +- Removing `google.api_core.gapic_v1.method.METRICS_METADATA_KEY`. It + can be accessed via + `google.api_core.gapic_v1.client_info.METRICS_METADATA_KEY` (#4588). + +### Dependencies + +- Upgrading to latest `grpcio==1.8.2` (#4642). For details, see + related gRPC [bug](https://github.com/grpc/grpc/issues/9688) + and [fix](https://github.com/grpc/grpc/pull/13665). + +PyPI: https://pypi.org/project/google-api-core/0.1.3/ + +## 0.1.2 + +- Upgrading `concurrent.futures` backport from `>= 3.0.0` + to `>= 3.2.0` (#4521). +- Moved `datetime`-related helpers from `google.cloud.core` to + `google.api_core.datetime_helpers` (#4399). +- Added missing `client_info` to `gapic_v1/__init__.py`'s + `__all__` (#4567). +- Added helpers for routing headers to `gapic_v1` (#4336). + +PyPI: https://pypi.org/project/google-api-core/0.1.2/ + +## 0.1.1 + +### Dependencies + +- Upgrading `grpcio` dependency from `1.2.0, < 1.6dev` to `>= 1.7.0` (#4280) + +PyPI: https://pypi.org/project/google-api-core/0.1.1/ + +## 0.1.0 + +Initial release + +Prior to being separated, this package was developed in `google-cloud-core`, so +relevant changes from that package are included here. + +- Add google.api.core.gapic_v1.config (#4022) +- Add google.api.core.helpers.grpc_helpers (#4041) +- Add google.api.core.gapic_v1.method (#4057) +- Add wrap_with_paging (#4067) +- Add grpc_helpers.create_channel (#4069) +- Add DEFAULT sentinel for gapic_v1.method (#4079) +- Remove `googleapis-common-protos` from deps in non-`core` packages. (#4098) +- Add google.api.core.operations_v1 (#4081) +- Fix test assertion in test_wrap_method_with_overriding_retry_deadline (#4131) +- Add google.api.core.helpers.general_helpers.wraps (#4166) +- Update Docs with Python Setup Guide (#4187) +- Move modules in google.api.core.helpers up one level, delete google.api.core.helpers. (#4196) +- Clarify that PollingFuture timeout is in seconds. (#4201) +- Add api_core package (#4210) +- Replace usage of google.api.core with google.api_core (#4221) +- Add google.api_core.gapic_v2.client_info (#4225) +- Fix how api_core.operation populates exception errors (#4231) +- Fix bare except (#4250) +- Fix parsing of API errors with Unicode err message (#4251) +- Port gax proto helper methods (#4249) +- Remove gapic_v1.method.wrap_with_paging (#4257) +- Add final set of protobuf helpers to api_core (#4259) + +PyPI: https://pypi.org/project/google-api-core/0.1.0/ diff --git a/api_core/LICENSE b/api_core/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/api_core/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/api_core/MANIFEST.in b/api_core/MANIFEST.in new file mode 100644 index 000000000000..1fbc0d0b321e --- /dev/null +++ b/api_core/MANIFEST.in @@ -0,0 +1,3 @@ +include README.rst LICENSE +recursive-include tests * +global-exclude *.pyc __pycache__ diff --git a/api_core/README.rst b/api_core/README.rst new file mode 100644 index 000000000000..244043ea61b3 --- /dev/null +++ b/api_core/README.rst @@ -0,0 +1,23 @@ +Core Library for Google Client Libraries +======================================== + +|pypi| |versions| + +This library is not meant to stand-alone. Instead it defines +common helpers used by all Google API clients. For more information, see the +`documentation`_. + +.. |pypi| image:: https://img.shields.io/pypi/v/google-api_core.svg + :target: https://pypi.org/project/google-api_core/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/google-api_core.svg + :target: https://pypi.org/project/google-api_core/ +.. _documentation: https://googleapis.dev/python/google-api-core/latest + + +Supported Python Versions +------------------------- +Python >= 3.5 + +Deprecated Python Versions +-------------------------- +Python == 2.7. Python 2.7 support will be removed on January 1, 2020. diff --git a/api_core/docs/auth.rst b/api_core/docs/auth.rst new file mode 100644 index 000000000000..f309a3a018ec --- /dev/null +++ b/api_core/docs/auth.rst @@ -0,0 +1,301 @@ +Authentication +************** + +.. _Overview: + +Overview +======== + +* **If you're running in Compute Engine or App Engine**, + authentication should "just work". + +* **If you're developing locally**, + the easiest way to authenticate is using the `Google Cloud SDK`_: + + .. code-block:: bash + + $ gcloud auth application-default login + + Note that this command generates credentials for client libraries. To authenticate the CLI itself, use: + + .. code-block:: bash + + $ gcloud auth login + + Previously, ``gcloud auth login`` was used for both use cases. If + your ``gcloud`` installation does not support the new command, + please update it: + + .. code-block:: bash + + $ gcloud components update + +.. _Google Cloud SDK: http://cloud.google.com/sdk + + +* **If you're running your application elsewhere**, + you should download a `service account`_ JSON keyfile + and point to it using an environment variable: + + .. code-block:: bash + + $ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json" + +.. _service account: https://cloud.google.com/storage/docs/authentication#generating-a-private-key + +Client-Provided Authentication +============================== + +Every package uses a :class:`Client ` +as a base for interacting with an API. +For example: + +.. code-block:: python + + from google.cloud import datastore + client = datastore.Client() + +Passing no arguments at all will "just work" if you've followed the +instructions in the :ref:`Overview`. +The credentials are inferred from your local environment by using +Google `Application Default Credentials`_. + +.. _Application Default Credentials: https://developers.google.com/identity/protocols/application-default-credentials + +.. _Precedence: + +Credential Discovery Precedence +------------------------------- + +When loading the `Application Default Credentials`_, +the library will check for credentials in your environment by following the +precedence outlined by :func:`google.auth.default`. + +Explicit Credentials +==================== + +The Application Default Credentials discussed above can be useful +if your code needs to run in many different environments or +if you just don't want authentication to be a focus in your code. + +However, you may want to be explicit because + +* your code will only run in one place +* you may have code which needs to be run as a specific service account + every time (rather than with the locally inferred credentials) +* you may want to use two separate accounts to simultaneously access data + from different projects + +In these situations, you can create an explicit +:class:`~google.auth.credentials.Credentials` object suited to your environment. +After creation, you can pass it directly to a :class:`Client `: + +.. code:: python + + client = Client(credentials=credentials) + +.. tip:: + To create a credentials object, follow the `google-auth-guide`_. + +.. _google-auth-guide: https://google-auth.readthedocs.io/en/latest/user-guide.html#service-account-private-key-files + + +Google App Engine Environment +----------------------------- + +To create +:class:`credentials ` +just for Google App Engine: + +.. code:: python + + from google.auth import app_engine + credentials = app_engine.Credentials() + +Google Compute Engine Environment +--------------------------------- + +To create +:class:`credentials ` +just for Google Compute Engine: + +.. code:: python + + from google.auth import compute_engine + credentials = compute_engine.Credentials() + +Service Accounts +---------------- + +A `service account`_ is stored in a JSON keyfile. + +The +:meth:`from_service_account_json() ` +factory can be used to create a :class:`Client ` with +service account credentials. + +For example, with a JSON keyfile: + +.. code:: python + + client = Client.from_service_account_json('/path/to/keyfile.json') + +.. tip:: + + Previously the Google Cloud Console would issue a PKCS12/P12 key for your + service account. This library does not support that key format. You can + generate a new JSON key for the same service account from the console. + +User Accounts (3-legged OAuth 2.0) with a refresh token +------------------------------------------------------- + +The majority of cases are intended to authenticate machines or +workers rather than actual user accounts. However, it's also +possible to call Google Cloud APIs with a user account via +`OAuth 2.0`_. + +.. _OAuth 2.0: https://developers.google.com/identity/protocols/OAuth2 + +.. tip:: + + A production application should **use a service account**, + but you may wish to use your own personal user account when first + getting started with the ``google-cloud-python`` library. + +The simplest way to use credentials from a user account is via +Application Default Credentials using ``gcloud auth login`` +(as mentioned above) and :func:`google.auth.default`: + +.. code:: python + + import google.auth + + credentials, project = google.auth.default() + +This will still follow the :ref:`precedence ` +described above, +so be sure none of the other possible environments conflict +with your user provided credentials. + +Advanced users of `oauth2client`_ can also use custom flows to +create credentials using `client secrets`_ or using a +`webserver flow`_. +After creation, :class:`Credentials ` +can be serialized with +:meth:`to_json() ` +and stored in a file and then and deserialized with +:meth:`from_json() `. In order +to use ``oauth2client``'s credentials with this library, you'll need to +`convert them`_. + +.. _oauth2client: https://github.com/Google/oauth2client. +.. _client secrets: https://developers.google.com/api-client-library/python/guide/aaa_oauth#flow_from_clientsecrets +.. _webserver flow: https://developers.google.com/api-client-library/python/guide/aaa_oauth#OAuth2WebServerFlow +.. _convert them: http://google-auth.readthedocs.io/en/stable/user-guide.html#user-credentials + +Troubleshooting +=============== + +Setting up a Service Account +---------------------------- + +If your application is not running on Google Compute Engine, +you need a `Google Developers Service Account`_. + +#. Visit the `Google Developers Console`_. + +#. Create a new project or click on an existing project. + +#. Navigate to **APIs & auth** > **APIs** and enable the APIs + that your application requires. + + .. raw:: html + + + + .. note:: + + You may need to enable billing in order to use these services. + + * **BigQuery** + + * BigQuery API + + * **Datastore** + + * Google Cloud Datastore API + + * **Pub/Sub** + + * Google Cloud Pub/Sub + + * **Storage** + + * Google Cloud Storage + * Google Cloud Storage JSON API + +#. Navigate to **APIs & auth** > **Credentials**. + + You should see a screen like one of the following: + + .. raw:: html + + + + .. raw:: html + + + + Find the "Add credentials" drop down and select "Service account" to be + guided through downloading a new JSON keyfile. + + If you want to re-use an existing service account, + you can easily generate a new keyfile. + Just select the account you wish to re-use, + and click **Generate new JSON key**: + + .. raw:: html + + + +.. _Google Developers Console: https://console.developers.google.com/project +.. _Google Developers Service Account: https://developers.google.com/accounts/docs/OAuth2ServiceAccount + +Using Google Compute Engine +--------------------------- + +If your code is running on Google Compute Engine, +using the inferred Google `Application Default Credentials`_ +will be sufficient for retrieving credentials. + +However, by default your credentials may not grant you +access to the services you intend to use. +Be sure when you `set up the GCE instance`_, +you add the correct scopes for the APIs you want to access: + +* **All APIs** + + * ``https://www.googleapis.com/auth/cloud-platform`` + * ``https://www.googleapis.com/auth/cloud-platform.read-only`` + +* **BigQuery** + + * ``https://www.googleapis.com/auth/bigquery`` + * ``https://www.googleapis.com/auth/bigquery.insertdata`` + +* **Datastore** + + * ``https://www.googleapis.com/auth/datastore`` + * ``https://www.googleapis.com/auth/userinfo.email`` + +* **Pub/Sub** + + * ``https://www.googleapis.com/auth/pubsub`` + +* **Storage** + + * ``https://www.googleapis.com/auth/devstorage.full_control`` + * ``https://www.googleapis.com/auth/devstorage.read_only`` + * ``https://www.googleapis.com/auth/devstorage.read_write`` + +.. _set up the GCE instance: https://cloud.google.com/compute/docs/authentication#using diff --git a/api_core/docs/changelog.md b/api_core/docs/changelog.md new file mode 120000 index 000000000000..04c99a55caae --- /dev/null +++ b/api_core/docs/changelog.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/api_core/docs/client_info.rst b/api_core/docs/client_info.rst new file mode 100644 index 000000000000..e976b1863c7c --- /dev/null +++ b/api_core/docs/client_info.rst @@ -0,0 +1,11 @@ +Client Information Helpers +========================== + +.. automodule:: google.api_core.client_info + :members: + :show-inheritance: + +.. automodule:: google.api_core.gapic_v1.client_info + :members: + :show-inheritance: + diff --git a/api_core/docs/client_options.rst b/api_core/docs/client_options.rst new file mode 100644 index 000000000000..da7c9a339bbd --- /dev/null +++ b/api_core/docs/client_options.rst @@ -0,0 +1,6 @@ +Client Options +========================== + +.. automodule:: google.api_core.client_options + :members: + :show-inheritance: diff --git a/api_core/docs/conf.py b/api_core/docs/conf.py new file mode 100644 index 000000000000..60b1f7a3f0b9 --- /dev/null +++ b/api_core/docs/conf.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- +# +# google-api-core documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "1.6.3" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = u"google-api-core" +copyright = u"2017, Google" +author = u"Google APIs" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = [] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-api-core-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "google-api-core.tex", + u"google-api-core Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, "google-api-core", u"google-api-core Documentation", [author], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "google-api-core", + u"google-api-core Documentation", + author, + "google-api-core", + "GAPIC library for the {metadata.shortName} v1beta1 service", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("https://2.python-requests.org/en/master/", None), + "fastavro": ("https://fastavro.readthedocs.io/en/stable/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/api_core/docs/exceptions.rst b/api_core/docs/exceptions.rst new file mode 100644 index 000000000000..d671f4e02046 --- /dev/null +++ b/api_core/docs/exceptions.rst @@ -0,0 +1,6 @@ +Exceptions +========== + +.. automodule:: google.api_core.exceptions + :members: + :show-inheritance: diff --git a/api_core/docs/futures.rst b/api_core/docs/futures.rst new file mode 100644 index 000000000000..7a43da9de291 --- /dev/null +++ b/api_core/docs/futures.rst @@ -0,0 +1,10 @@ +Futures +======================= + +.. automodule:: google.api_core.future + :members: + :show-inheritance: + +.. automodule:: google.api_core.future.polling + :members: + :show-inheritance: \ No newline at end of file diff --git a/api_core/docs/helpers.rst b/api_core/docs/helpers.rst new file mode 100644 index 000000000000..6f72df9561ef --- /dev/null +++ b/api_core/docs/helpers.rst @@ -0,0 +1,26 @@ +Helpers +======= + + +General Helpers +--------------- + +.. automodule:: google.api_core.general_helpers + :members: + :show-inheritance: + + +Datetime Helpers +---------------- + +.. automodule:: google.api_core.datetime_helpers + :members: + :show-inheritance: + + +gRPC Helpers +------------ + +.. automodule:: google.api_core.grpc_helpers + :members: + :show-inheritance: diff --git a/api_core/docs/iam.rst b/api_core/docs/iam.rst new file mode 100644 index 000000000000..bb80ae3c5798 --- /dev/null +++ b/api_core/docs/iam.rst @@ -0,0 +1,7 @@ +Identity and Access Management +============================== + +.. automodule:: google.api_core.iam + :members: + :show-inheritance: + :member-order: bysource diff --git a/api_core/docs/index.rst b/api_core/docs/index.rst new file mode 100644 index 000000000000..67572a0bbf96 --- /dev/null +++ b/api_core/docs/index.rst @@ -0,0 +1,30 @@ +The ``google-cloud-core`` package contains helpers common to all +``google-cloud-*`` packages. In an attempt to reach a stable API, +much of the functionality has been split out into this package, +``google-api-core``. + +Core +==== + +.. toctree:: + auth + client_info + client_options + exceptions + futures + helpers + iam + operation + operations_client + page_iterator + path_template + retry + timeout + +Changelog +~~~~~~~~~ + +.. toctree:: + :maxdepth: 2 + + changelog diff --git a/api_core/docs/operation.rst b/api_core/docs/operation.rst new file mode 100644 index 000000000000..c5e676621318 --- /dev/null +++ b/api_core/docs/operation.rst @@ -0,0 +1,6 @@ +Long-Running Operations +======================= + +.. automodule:: google.api_core.operation + :members: + :show-inheritance: diff --git a/api_core/docs/operations_client.rst b/api_core/docs/operations_client.rst new file mode 100644 index 000000000000..be466d3fbf43 --- /dev/null +++ b/api_core/docs/operations_client.rst @@ -0,0 +1,6 @@ +Long-Running Operations Client +============================== + +.. automodule:: google.api_core.operations_v1 + :members: + :show-inheritance: diff --git a/api_core/docs/page_iterator.rst b/api_core/docs/page_iterator.rst new file mode 100644 index 000000000000..28842da2a76d --- /dev/null +++ b/api_core/docs/page_iterator.rst @@ -0,0 +1,6 @@ +Page Iterators +============== + +.. automodule:: google.api_core.page_iterator + :members: + :show-inheritance: diff --git a/api_core/docs/path_template.rst b/api_core/docs/path_template.rst new file mode 100644 index 000000000000..220779e3ba9f --- /dev/null +++ b/api_core/docs/path_template.rst @@ -0,0 +1,6 @@ +Path Templates +============== + +.. automodule:: google.api_core.path_template + :members: + :show-inheritance: diff --git a/api_core/docs/retry.rst b/api_core/docs/retry.rst new file mode 100644 index 000000000000..23a7d70f3431 --- /dev/null +++ b/api_core/docs/retry.rst @@ -0,0 +1,6 @@ +Retry +===== + +.. automodule:: google.api_core.retry + :members: + :show-inheritance: diff --git a/api_core/docs/timeout.rst b/api_core/docs/timeout.rst new file mode 100644 index 000000000000..943d42508997 --- /dev/null +++ b/api_core/docs/timeout.rst @@ -0,0 +1,6 @@ +Timeout +======= + +.. automodule:: google.api_core.timeout + :members: + :show-inheritance: diff --git a/api_core/google/__init__.py b/api_core/google/__init__.py new file mode 100644 index 000000000000..0d0a4c3ab273 --- /dev/null +++ b/api_core/google/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google namespace package.""" + +try: + import pkg_resources + + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/api_core/google/api_core/__init__.py b/api_core/google/api_core/__init__.py new file mode 100644 index 000000000000..c762e18309db --- /dev/null +++ b/api_core/google/api_core/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google API Core. + +This package contains common code and utilties used by Google client libraries. +""" + +from pkg_resources import get_distribution + + +__version__ = get_distribution("google-api-core").version diff --git a/api_core/google/api_core/bidi.py b/api_core/google/api_core/bidi.py new file mode 100644 index 000000000000..f73c7c9dfabc --- /dev/null +++ b/api_core/google/api_core/bidi.py @@ -0,0 +1,727 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bi-directional streaming RPC helpers.""" + +import collections +import datetime +import logging +import threading +import time + +from six.moves import queue + +from google.api_core import exceptions + +_LOGGER = logging.getLogger(__name__) +_BIDIRECTIONAL_CONSUMER_NAME = "Thread-ConsumeBidirectionalStream" + + +class _RequestQueueGenerator(object): + """A helper for sending requests to a gRPC stream from a Queue. + + This generator takes requests off a given queue and yields them to gRPC. + + This helper is useful when you have an indeterminate, indefinite, or + otherwise open-ended set of requests to send through a request-streaming + (or bidirectional) RPC. + + The reason this is necessary is because gRPC takes an iterator as the + request for request-streaming RPCs. gRPC consumes this iterator in another + thread to allow it to block while generating requests for the stream. + However, if the generator blocks indefinitely gRPC will not be able to + clean up the thread as it'll be blocked on `next(iterator)` and not be able + to check the channel status to stop iterating. This helper mitigates that + by waiting on the queue with a timeout and checking the RPC state before + yielding. + + Finally, it allows for retrying without swapping queues because if it does + pull an item off the queue when the RPC is inactive, it'll immediately put + it back and then exit. This is necessary because yielding the item in this + case will cause gRPC to discard it. In practice, this means that the order + of messages is not guaranteed. If such a thing is necessary it would be + easy to use a priority queue. + + Example:: + + requests = request_queue_generator(q) + call = stub.StreamingRequest(iter(requests)) + requests.call = call + + for response in call: + print(response) + q.put(...) + + Note that it is possible to accomplish this behavior without "spinning" + (using a queue timeout). One possible way would be to use more threads to + multiplex the grpc end event with the queue, another possible way is to + use selectors and a custom event/queue object. Both of these approaches + are significant from an engineering perspective for small benefit - the + CPU consumed by spinning is pretty minuscule. + + Args: + queue (queue.Queue): The request queue. + period (float): The number of seconds to wait for items from the queue + before checking if the RPC is cancelled. In practice, this + determines the maximum amount of time the request consumption + thread will live after the RPC is cancelled. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is done independently of the request queue to allow fo + easily restarting streams that require some initial configuration + request. + """ + + def __init__(self, queue, period=1, initial_request=None): + self._queue = queue + self._period = period + self._initial_request = initial_request + self.call = None + + def _is_active(self): + # Note: there is a possibility that this starts *before* the call + # property is set. So we have to check if self.call is set before + # seeing if it's active. + if self.call is not None and not self.call.is_active(): + return False + else: + return True + + def __iter__(self): + if self._initial_request is not None: + if callable(self._initial_request): + yield self._initial_request() + else: + yield self._initial_request + + while True: + try: + item = self._queue.get(timeout=self._period) + except queue.Empty: + if not self._is_active(): + _LOGGER.debug( + "Empty queue and inactive call, exiting request " "generator." + ) + return + else: + # call is still active, keep waiting for queue items. + continue + + # The consumer explicitly sent "None", indicating that the request + # should end. + if item is None: + _LOGGER.debug("Cleanly exiting request generator.") + return + + if not self._is_active(): + # We have an item, but the call is closed. We should put the + # item back on the queue so that the next call can consume it. + self._queue.put(item) + _LOGGER.debug( + "Inactive call, replacing item on queue and exiting " + "request generator." + ) + return + + yield item + + +class _Throttle(object): + """A context manager limiting the total entries in a sliding time window. + + If more than ``access_limit`` attempts are made to enter the context manager + instance in the last ``time window`` interval, the exceeding requests block + until enough time elapses. + + The context manager instances are thread-safe and can be shared between + multiple threads. If multiple requests are blocked and waiting to enter, + the exact order in which they are allowed to proceed is not determined. + + Example:: + + max_three_per_second = _Throttle( + access_limit=3, time_window=datetime.timedelta(seconds=1) + ) + + for i in range(5): + with max_three_per_second as time_waited: + print("{}: Waited {} seconds to enter".format(i, time_waited)) + + Args: + access_limit (int): the maximum number of entries allowed in the time window + time_window (datetime.timedelta): the width of the sliding time window + """ + + def __init__(self, access_limit, time_window): + if access_limit < 1: + raise ValueError("access_limit argument must be positive") + + if time_window <= datetime.timedelta(0): + raise ValueError("time_window argument must be a positive timedelta") + + self._time_window = time_window + self._access_limit = access_limit + self._past_entries = collections.deque(maxlen=access_limit) # least recent first + self._entry_lock = threading.Lock() + + def __enter__(self): + with self._entry_lock: + cutoff_time = datetime.datetime.now() - self._time_window + + # drop the entries that are too old, as they are no longer relevant + while self._past_entries and self._past_entries[0] < cutoff_time: + self._past_entries.popleft() + + if len(self._past_entries) < self._access_limit: + self._past_entries.append(datetime.datetime.now()) + return 0.0 # no waiting was needed + + to_wait = (self._past_entries[0] - cutoff_time).total_seconds() + time.sleep(to_wait) + + self._past_entries.append(datetime.datetime.now()) + return to_wait + + def __exit__(self, *_): + pass + + def __repr__(self): + return "{}(access_limit={}, time_window={})".format( + self.__class__.__name__, + self._access_limit, + repr(self._time_window), + ) + + +class BidiRpc(object): + """A helper for consuming a bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + Example:: + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + rpc = BidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + metadata=[('name', 'value')] + ) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + metadata (Sequence[Tuple(str, str)]): RPC metadata to include in + the request. + """ + + def __init__(self, start_rpc, initial_request=None, metadata=None): + self._start_rpc = start_rpc + self._initial_request = initial_request + self._rpc_metadata = metadata + self._request_queue = queue.Queue() + self._request_generator = None + self._is_active = False + self._callbacks = [] + self.call = None + + def add_done_callback(self, callback): + """Adds a callback that will be called when the RPC terminates. + + This occurs when the RPC errors or is successfully terminated. + + Args: + callback (Callable[[grpc.Future], None]): The callback to execute. + It will be provided with the same gRPC future as the underlying + stream which will also be a :class:`grpc.Call`. + """ + self._callbacks.append(callback) + + def _on_call_done(self, future): + for callback in self._callbacks: + callback(future) + + def open(self): + """Opens the stream.""" + if self.is_active: + raise ValueError("Can not open an already open stream.") + + request_generator = _RequestQueueGenerator( + self._request_queue, initial_request=self._initial_request + ) + call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + + request_generator.call = call + + # TODO: api_core should expose the future interface for wrapped + # callables as well. + if hasattr(call, "_wrapped"): # pragma: NO COVER + call._wrapped.add_done_callback(self._on_call_done) + else: + call.add_done_callback(self._on_call_done) + + self._request_generator = request_generator + self.call = call + + def close(self): + """Closes the stream.""" + if self.call is None: + return + + self._request_queue.put(None) + self.call.cancel() + self._request_generator = None + # Don't set self.call to None. Keep it around so that send/recv can + # raise the error. + + def send(self, request): + """Queue a message to be sent on the stream. + + Send is non-blocking. + + If the underlying RPC has been closed, this will raise. + + Args: + request (protobuf.Message): The request to send. + """ + if self.call is None: + raise ValueError("Can not send() on an RPC that has never been open()ed.") + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if self.call.is_active(): + self._request_queue.put(request) + else: + # calling next should cause the call to raise. + next(self.call) + + def recv(self): + """Wait for a message to be returned from the stream. + + Recv is blocking. + + If the underlying RPC has been closed, this will raise. + + Returns: + protobuf.Message: The received message. + """ + if self.call is None: + raise ValueError("Can not recv() on an RPC that has never been open()ed.") + + return next(self.call) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + return self.call is not None and self.call.is_active() + + @property + def pending_requests(self): + """int: Returns an estimate of the number of queued requests.""" + return self._request_queue.qsize() + + +def _never_terminate(future_or_error): + """By default, no errors cause BiDi termination.""" + return False + + +class ResumableBidiRpc(BidiRpc): + """A :class:`BidiRpc` that can automatically resume the stream on errors. + + It uses the ``should_recover`` arg to determine if it should re-establish + the stream on error. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + metadata = [('header_name', 'value')] + + rpc = ResumableBidiRpc( + stub.StreamingRpc, + should_recover=should_recover, + initial_request=initial_request, + metadata=metadata + ) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + should_recover (Callable[[Exception], bool]): A function that returns + True if the stream should be recovered. This will be called + whenever an error is encountered on the stream. + should_terminate (Callable[[Exception], bool]): A function that returns + True if the stream should be terminated. This will be called + whenever an error is encountered on the stream. + metadata Sequence[Tuple(str, str)]: RPC metadata to include in + the request. + throttle_reopen (bool): If ``True``, throttling will be applied to + stream reopen calls. Defaults to ``False``. + """ + + def __init__( + self, + start_rpc, + should_recover, + should_terminate=_never_terminate, + initial_request=None, + metadata=None, + throttle_reopen=False, + ): + super(ResumableBidiRpc, self).__init__(start_rpc, initial_request, metadata) + self._should_recover = should_recover + self._should_terminate = should_terminate + self._operational_lock = threading.RLock() + self._finalized = False + self._finalize_lock = threading.Lock() + + if throttle_reopen: + self._reopen_throttle = _Throttle( + access_limit=5, time_window=datetime.timedelta(seconds=10), + ) + else: + self._reopen_throttle = None + + def _finalize(self, result): + with self._finalize_lock: + if self._finalized: + return + + for callback in self._callbacks: + callback(result) + + self._finalized = True + + def _on_call_done(self, future): + # Unlike the base class, we only execute the callbacks on a terminal + # error, not for errors that we can recover from. Note that grpc's + # "future" here is also a grpc.RpcError. + with self._operational_lock: + if self._should_terminate(future): + self._finalize(future) + elif not self._should_recover(future): + self._finalize(future) + else: + _LOGGER.debug("Re-opening stream from gRPC callback.") + self._reopen() + + def _reopen(self): + with self._operational_lock: + # Another thread already managed to re-open this stream. + if self.call is not None and self.call.is_active(): + _LOGGER.debug("Stream was already re-established.") + return + + self.call = None + # Request generator should exit cleanly since the RPC its bound to + # has exited. + self._request_generator = None + + # Note: we do not currently do any sort of backoff here. The + # assumption is that re-establishing the stream under normal + # circumstances will happen in intervals greater than 60s. + # However, it is possible in a degenerative case that the server + # closes the stream rapidly which would lead to thrashing here, + # but hopefully in those cases the server would return a non- + # retryable error. + + try: + if self._reopen_throttle: + with self._reopen_throttle: + self.open() + else: + self.open() + # If re-opening or re-calling the method fails for any reason, + # consider it a terminal error and finalize the stream. + except Exception as exc: + _LOGGER.debug("Failed to re-open stream due to %s", exc) + self._finalize(exc) + raise + + _LOGGER.info("Re-established stream") + + def _recoverable(self, method, *args, **kwargs): + """Wraps a method to recover the stream and retry on error. + + If a retryable error occurs while making the call, then the stream will + be re-opened and the method will be retried. This happens indefinitely + so long as the error is a retryable one. If an error occurs while + re-opening the stream, then this method will raise immediately and + trigger finalization of this object. + + Args: + method (Callable[..., Any]): The method to call. + args: The args to pass to the method. + kwargs: The kwargs to pass to the method. + """ + while True: + try: + return method(*args, **kwargs) + + except Exception as exc: + with self._operational_lock: + _LOGGER.debug("Call to retryable %r caused %s.", method, exc) + + if self._should_terminate(exc): + self.close() + _LOGGER.debug("Terminating %r due to %s.", method, exc) + self._finalize(exc) + break + + if not self._should_recover(exc): + self.close() + _LOGGER.debug("Not retrying %r due to %s.", method, exc) + self._finalize(exc) + raise exc + + _LOGGER.debug("Re-opening stream from retryable %r.", method) + self._reopen() + + def _send(self, request): + # Grab a reference to the RPC call. Because another thread (notably + # the gRPC error thread) can modify self.call (by invoking reopen), + # we should ensure our reference can not change underneath us. + # If self.call is modified (such as replaced with a new RPC call) then + # this will use the "old" RPC, which should result in the same + # exception passed into gRPC's error handler being raised here, which + # will be handled by the usual error handling in retryable. + with self._operational_lock: + call = self.call + + if call is None: + raise ValueError("Can not send() on an RPC that has never been open()ed.") + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if call.is_active(): + self._request_queue.put(request) + pass + else: + # calling next should cause the call to raise. + next(call) + + def send(self, request): + return self._recoverable(self._send, request) + + def _recv(self): + with self._operational_lock: + call = self.call + + if call is None: + raise ValueError("Can not recv() on an RPC that has never been open()ed.") + + return next(call) + + def recv(self): + return self._recoverable(self._recv) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + # Use the operational lock. It's entirely possible for something + # to check the active state *while* the RPC is being retried. + # Also, use finalized to track the actual terminal state here. + # This is because if the stream is re-established by the gRPC thread + # it's technically possible to check this between when gRPC marks the + # RPC as inactive and when gRPC executes our callback that re-opens + # the stream. + with self._operational_lock: + return self.call is not None and not self._finalized + + +class BackgroundConsumer(object): + """A bi-directional stream consumer that runs in a separate thread. + + This maps the consumption of a stream into a callback-based model. It also + provides :func:`pause` and :func:`resume` to allow for flow-control. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + def on_response(response): + print(response) + + consumer = BackgroundConsumer(rpc, on_response) + consumer.start() + + Note that error handling *must* be done by using the provided + ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit + whenever the RPC itself exits and will not provide any error details. + + Args: + bidi_rpc (BidiRpc): The RPC to consume. Should not have been + ``open()``ed yet. + on_response (Callable[[protobuf.Message], None]): The callback to + be called for every response on the stream. + """ + + def __init__(self, bidi_rpc, on_response): + self._bidi_rpc = bidi_rpc + self._on_response = on_response + self._paused = False + self._wake = threading.Condition() + self._thread = None + self._operational_lock = threading.Lock() + + def _on_call_done(self, future): + # Resume the thread if it's paused, this prevents blocking forever + # when the RPC has terminated. + self.resume() + + def _thread_main(self, ready): + try: + ready.set() + self._bidi_rpc.add_done_callback(self._on_call_done) + self._bidi_rpc.open() + + while self._bidi_rpc.is_active: + # Do not allow the paused status to change at all during this + # section. There is a condition where we could be resumed + # between checking if we are paused and calling wake.wait(), + # which means that we will miss the notification to wake up + # (oops!) and wait for a notification that will never come. + # Keeping the lock throughout avoids that. + # In the future, we could use `Condition.wait_for` if we drop + # Python 2.7. + with self._wake: + while self._paused: + _LOGGER.debug("paused, waiting for waking.") + self._wake.wait() + _LOGGER.debug("woken.") + + _LOGGER.debug("waiting for recv.") + response = self._bidi_rpc.recv() + _LOGGER.debug("recved response.") + self._on_response(response) + + except exceptions.GoogleAPICallError as exc: + _LOGGER.debug( + "%s caught error %s and will exit. Generally this is due to " + "the RPC itself being cancelled and the error will be " + "surfaced to the calling code.", + _BIDIRECTIONAL_CONSUMER_NAME, + exc, + exc_info=True, + ) + + except Exception as exc: + _LOGGER.exception( + "%s caught unexpected exception %s and will exit.", + _BIDIRECTIONAL_CONSUMER_NAME, + exc, + ) + + _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) + + def start(self): + """Start the background thread and begin consuming the thread.""" + with self._operational_lock: + ready = threading.Event() + thread = threading.Thread( + name=_BIDIRECTIONAL_CONSUMER_NAME, + target=self._thread_main, + args=(ready,), + ) + thread.daemon = True + thread.start() + # Other parts of the code rely on `thread.is_alive` which + # isn't sufficient to know if a thread is active, just that it may + # soon be active. This can cause races. Further protect + # against races by using a ready event and wait on it to be set. + ready.wait() + self._thread = thread + _LOGGER.debug("Started helper thread %s", thread.name) + + def stop(self): + """Stop consuming the stream and shutdown the background thread.""" + with self._operational_lock: + self._bidi_rpc.close() + + if self._thread is not None: + # Resume the thread to wake it up in case it is sleeping. + self.resume() + self._thread.join() + + self._thread = None + + @property + def is_active(self): + """bool: True if the background thread is active.""" + return self._thread is not None and self._thread.is_alive() + + def pause(self): + """Pauses the response stream. + + This does *not* pause the request stream. + """ + with self._wake: + self._paused = True + + def resume(self): + """Resumes the response stream.""" + with self._wake: + self._paused = False + self._wake.notifyAll() + + @property + def is_paused(self): + """bool: True if the response stream is paused.""" + return self._paused diff --git a/api_core/google/api_core/client_info.py b/api_core/google/api_core/client_info.py new file mode 100644 index 000000000000..b196b7a987e4 --- /dev/null +++ b/api_core/google/api_core/client_info.py @@ -0,0 +1,96 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for providing client information. + +Client information is used to send information about the calling client, +such as the library and Python version, to API services. +""" + +import platform + +import pkg_resources + +_PY_VERSION = platform.python_version() +_API_CORE_VERSION = pkg_resources.get_distribution("google-api-core").version + +try: + _GRPC_VERSION = pkg_resources.get_distribution("grpcio").version +except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GRPC_VERSION = None + + +class ClientInfo(object): + """Client information used to generate a user-agent for API calls. + + This user-agent information is sent along with API calls to allow the + receiving service to do analytics on which versions of Python and Google + libraries are being used. + + Args: + python_version (str): The Python interpreter version, for example, + ``'2.7.13'``. + grpc_version (Optional[str]): The gRPC library version. + api_core_version (str): The google-api-core library version. + gapic_version (Optional[str]): The sversion of gapic-generated client + library, if the library was generated by gapic. + client_library_version (Optional[str]): The version of the client + library, generally used if the client library was not generated + by gapic or if additional functionality was built on top of + a gapic client library. + user_agent (Optional[str]): Prefix to the user agent header. This is + used to supply information such as application name or partner tool. + Recommended format: ``application-or-tool-ID/major.minor.version``. + """ + + def __init__( + self, + python_version=_PY_VERSION, + grpc_version=_GRPC_VERSION, + api_core_version=_API_CORE_VERSION, + gapic_version=None, + client_library_version=None, + user_agent=None, + ): + self.python_version = python_version + self.grpc_version = grpc_version + self.api_core_version = api_core_version + self.gapic_version = gapic_version + self.client_library_version = client_library_version + self.user_agent = user_agent + + def to_user_agent(self): + """Returns the user-agent string for this client info.""" + + # Note: the order here is important as the internal metrics system + # expects these items to be in specific locations. + ua = "" + + if self.user_agent is not None: + ua += "{user_agent} " + + ua += "gl-python/{python_version} " + + if self.grpc_version is not None: + ua += "grpc/{grpc_version} " + + ua += "gax/{api_core_version} " + + if self.gapic_version is not None: + ua += "gapic/{gapic_version} " + + if self.client_library_version is not None: + ua += "gccl/{client_library_version} " + + return ua.format(**self.__dict__).strip() diff --git a/api_core/google/api_core/client_options.py b/api_core/google/api_core/client_options.py new file mode 100644 index 000000000000..1144d2f62a37 --- /dev/null +++ b/api_core/google/api_core/client_options.py @@ -0,0 +1,69 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Client options class. + +Client options provide a consistent interface for user options to be defined +across clients. + +You can pass a client options object to a client. + +.. code-block:: python + + from google.api_core.client_options import ClientOptions + from google.cloud.vision_v1 import ImageAnnotatorClient + + options = ClientOptions(api_endpoint="foo.googleapis.com") + + client = ImageAnnotatorClient(client_options=options) + +You can also pass a dictionary. + +.. code-block:: python + + from google.cloud.vision_v1 import ImageAnnotatorClient + + client = ImageAnnotatorClient(client_options={"api_endpoint": "foo.googleapis.com"}) + + +""" + + +class ClientOptions(object): + """Client Options used to set options on clients. + + Args: + api_endpoint (str): The desired API endpoint, e.g., compute.googleapis.com + """ + + def __init__(self, api_endpoint=None): + self.api_endpoint = api_endpoint + + +def from_dict(options): + """Construct a client options object from a dictionary. + + Args: + options (dict): A dictionary with client options. + """ + + client_options = ClientOptions() + + for key, value in options.items(): + if hasattr(client_options, key): + setattr(client_options, key, value) + else: + raise ValueError("ClientOptions does not accept an option '" + key + "'") + + return client_options diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py new file mode 100644 index 000000000000..84c1bb7f512c --- /dev/null +++ b/api_core/google/api_core/datetime_helpers.py @@ -0,0 +1,303 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for :mod:`datetime`.""" + +import calendar +import datetime +import re + +import pytz + +from google.protobuf import timestamp_pb2 + + +_UTC_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) +_RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ" +_RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S" +# datetime.strptime cannot handle nanosecond precision: parse w/ regex +_RFC3339_NANOS = re.compile( + r""" + (?P + \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS + ) + ( # Optional decimal part + \. # decimal point + (?P\d{1,9}) # nanoseconds, maybe truncated + )? + Z # Zulu +""", + re.VERBOSE, +) + + +def utcnow(): + """A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests.""" + return datetime.datetime.utcnow() + + +def to_milliseconds(value): + """Convert a zone-aware datetime to milliseconds since the unix epoch. + + Args: + value (datetime.datetime): The datetime to covert. + + Returns: + int: Milliseconds since the unix epoch. + """ + micros = to_microseconds(value) + return micros // 1000 + + +def from_microseconds(value): + """Convert timestamp in microseconds since the unix epoch to datetime. + + Args: + value (float): The timestamp to convert, in microseconds. + + Returns: + datetime.datetime: The datetime object equivalent to the timestamp in + UTC. + """ + return _UTC_EPOCH + datetime.timedelta(microseconds=value) + + +def to_microseconds(value): + """Convert a datetime to microseconds since the unix epoch. + + Args: + value (datetime.datetime): The datetime to covert. + + Returns: + int: Microseconds since the unix epoch. + """ + if not value.tzinfo: + value = value.replace(tzinfo=pytz.utc) + # Regardless of what timezone is on the value, convert it to UTC. + value = value.astimezone(pytz.utc) + # Convert the datetime to a microsecond timestamp. + return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond + + +def from_iso8601_date(value): + """Convert a ISO8601 date string to a date. + + Args: + value (str): The ISO8601 date string. + + Returns: + datetime.date: A date equivalent to the date string. + """ + return datetime.datetime.strptime(value, "%Y-%m-%d").date() + + +def from_iso8601_time(value): + """Convert a zoneless ISO8601 time string to a time. + + Args: + value (str): The ISO8601 time string. + + Returns: + datetime.time: A time equivalent to the time string. + """ + return datetime.datetime.strptime(value, "%H:%M:%S").time() + + +def from_rfc3339(value): + """Convert a microsecond-precision timestamp to datetime. + + Args: + value (str): The RFC3339 string to convert. + + Returns: + datetime.datetime: The datetime object equivalent to the timestamp in + UTC. + """ + return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + + +def from_rfc3339_nanos(value): + """Convert a nanosecond-precision timestamp to a native datetime. + + .. note:: + Python datetimes do not support nanosecond precision; this function + therefore truncates such values to microseconds. + + Args: + value (str): The RFC3339 string to convert. + + Returns: + datetime.datetime: The datetime object equivalent to the timestamp in + UTC. + + Raises: + ValueError: If the timestamp does not match the RFC 3339 + regular expression. + """ + with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + raise ValueError( + "Timestamp: {!r}, does not match pattern: {!r}".format( + value, _RFC3339_NANOS.pattern + ) + ) + + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + + if fraction is None: + micros = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + micros = nanos // 1000 + + return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + + +def to_rfc3339(value, ignore_zone=True): + """Convert a datetime to an RFC3339 timestamp string. + + Args: + value (datetime.datetime): + The datetime object to be converted to a string. + ignore_zone (bool): If True, then the timezone (if any) of the + datetime object is ignored and the datetime is treated as UTC. + + Returns: + str: The RFC3339 formated string representing the datetime. + """ + if not ignore_zone and value.tzinfo is not None: + # Convert to UTC and remove the time zone info. + value = value.replace(tzinfo=None) - value.utcoffset() + + return value.strftime(_RFC3339_MICROS) + + +class DatetimeWithNanoseconds(datetime.datetime): + """Track nanosecond in addition to normal datetime attrs. + + Nanosecond can be passed only as a keyword argument. + """ + + __slots__ = ("_nanosecond",) + + # pylint: disable=arguments-differ + def __new__(cls, *args, **kw): + nanos = kw.pop("nanosecond", 0) + if nanos > 0: + if "microsecond" in kw: + raise TypeError("Specify only one of 'microsecond' or 'nanosecond'") + kw["microsecond"] = nanos // 1000 + inst = datetime.datetime.__new__(cls, *args, **kw) + inst._nanosecond = nanos or 0 + return inst + + # pylint: disable=arguments-differ + + @property + def nanosecond(self): + """Read-only: nanosecond precision.""" + return self._nanosecond + + def rfc3339(self): + """Return an RFC 3339-compliant timestamp. + + Returns: + (str): Timestamp string according to RFC 3339 spec. + """ + if self._nanosecond == 0: + return to_rfc3339(self) + nanos = str(self._nanosecond).rjust(9, '0').rstrip("0") + return "{}.{}Z".format(self.strftime(_RFC3339_NO_FRACTION), nanos) + + @classmethod + def from_rfc3339(cls, stamp): + """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + + Args: + stamp (str): RFC 3339 stamp, with up to nanosecond precision + + Returns: + :class:`DatetimeWithNanoseconds`: + an instance matching the timestamp string + + Raises: + ValueError: if `stamp` does not match the expected format + """ + with_nanos = _RFC3339_NANOS.match(stamp) + if with_nanos is None: + raise ValueError( + "Timestamp: {}, does not match pattern: {}".format( + stamp, _RFC3339_NANOS.pattern + ) + ) + bare = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + if fraction is None: + nanos = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + return cls( + bare.year, + bare.month, + bare.day, + bare.hour, + bare.minute, + bare.second, + nanosecond=nanos, + tzinfo=pytz.UTC, + ) + + def timestamp_pb(self): + """Return a timestamp message. + + Returns: + (:class:`~google.protobuf.timestamp_pb2.Timestamp`): Timestamp message + """ + inst = self if self.tzinfo is not None else self.replace(tzinfo=pytz.UTC) + delta = inst - _UTC_EPOCH + seconds = int(delta.total_seconds()) + nanos = self._nanosecond or self.microsecond * 1000 + return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + + @classmethod + def from_timestamp_pb(cls, stamp): + """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + + Args: + stamp (:class:`~google.protobuf.timestamp_pb2.Timestamp`): timestamp message + + Returns: + :class:`DatetimeWithNanoseconds`: + an instance matching the timestamp message + """ + microseconds = int(stamp.seconds * 1e6) + bare = from_microseconds(microseconds) + return cls( + bare.year, + bare.month, + bare.day, + bare.hour, + bare.minute, + bare.second, + nanosecond=stamp.nanos, + tzinfo=pytz.UTC, + ) diff --git a/api_core/google/api_core/exceptions.py b/api_core/google/api_core/exceptions.py new file mode 100644 index 000000000000..eed4ee40eee5 --- /dev/null +++ b/api_core/google/api_core/exceptions.py @@ -0,0 +1,462 @@ +# Copyright 2014 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exceptions raised by Google API core & clients. + +This module provides base classes for all errors raised by libraries based +on :mod:`google.api_core`, including both HTTP and gRPC clients. +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +from six.moves import http_client + +try: + import grpc +except ImportError: # pragma: NO COVER + grpc = None + +# Lookup tables for mapping exceptions from HTTP and gRPC transports. +# Populated by _APICallErrorMeta +_HTTP_CODE_TO_EXCEPTION = {} +_GRPC_CODE_TO_EXCEPTION = {} + + +class GoogleAPIError(Exception): + """Base class for all exceptions raised by Google API Clients.""" + + pass + + +@six.python_2_unicode_compatible +class RetryError(GoogleAPIError): + """Raised when a function has exhausted all of its available retries. + + Args: + message (str): The exception message. + cause (Exception): The last exception raised when retring the + function. + """ + + def __init__(self, message, cause): + super(RetryError, self).__init__(message) + self.message = message + self._cause = cause + + @property + def cause(self): + """The last exception raised when retrying the function.""" + return self._cause + + def __str__(self): + return "{}, last exception: {}".format(self.message, self.cause) + + +class _GoogleAPICallErrorMeta(type): + """Metaclass for registering GoogleAPICallError subclasses.""" + + def __new__(mcs, name, bases, class_dict): + cls = type.__new__(mcs, name, bases, class_dict) + if cls.code is not None: + _HTTP_CODE_TO_EXCEPTION.setdefault(cls.code, cls) + if cls.grpc_status_code is not None: + _GRPC_CODE_TO_EXCEPTION.setdefault(cls.grpc_status_code, cls) + return cls + + +@six.python_2_unicode_compatible +@six.add_metaclass(_GoogleAPICallErrorMeta) +class GoogleAPICallError(GoogleAPIError): + """Base class for exceptions raised by calling API methods. + + Args: + message (str): The exception message. + errors (Sequence[Any]): An optional list of error details. + response (Union[requests.Request, grpc.Call]): The response or + gRPC call metadata. + """ + + code = None + """Optional[int]: The HTTP status code associated with this error. + + This may be ``None`` if the exception does not have a direct mapping + to an HTTP error. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html + """ + + grpc_status_code = None + """Optional[grpc.StatusCode]: The gRPC status code associated with this + error. + + This may be ``None`` if the exception does not match up to a gRPC error. + """ + + def __init__(self, message, errors=(), response=None): + super(GoogleAPICallError, self).__init__(message) + self.message = message + """str: The exception message.""" + self._errors = errors + self._response = response + + def __str__(self): + return "{} {}".format(self.code, self.message) + + @property + def errors(self): + """Detailed error information. + + Returns: + Sequence[Any]: A list of additional error details. + """ + return list(self._errors) + + @property + def response(self): + """Optional[Union[requests.Request, grpc.Call]]: The response or + gRPC call metadata.""" + return self._response + + +class Redirection(GoogleAPICallError): + """Base class for for all redirection (HTTP 3xx) responses.""" + + +class MovedPermanently(Redirection): + """Exception mapping a ``301 Moved Permanently`` response.""" + + code = http_client.MOVED_PERMANENTLY + + +class NotModified(Redirection): + """Exception mapping a ``304 Not Modified`` response.""" + + code = http_client.NOT_MODIFIED + + +class TemporaryRedirect(Redirection): + """Exception mapping a ``307 Temporary Redirect`` response.""" + + code = http_client.TEMPORARY_REDIRECT + + +class ResumeIncomplete(Redirection): + """Exception mapping a ``308 Resume Incomplete`` response. + + .. note:: :attr:`http_client.PERMANENT_REDIRECT` is ``308``, but Google + APIs differ in their use of this status code. + """ + + code = 308 + + +class ClientError(GoogleAPICallError): + """Base class for all client error (HTTP 4xx) responses.""" + + +class BadRequest(ClientError): + """Exception mapping a ``400 Bad Request`` response.""" + + code = http_client.BAD_REQUEST + + +class InvalidArgument(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.INVALID_ARGUMENT` error.""" + + grpc_status_code = grpc.StatusCode.INVALID_ARGUMENT if grpc is not None else None + + +class FailedPrecondition(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.FAILED_PRECONDITION` + error.""" + + grpc_status_code = grpc.StatusCode.FAILED_PRECONDITION if grpc is not None else None + + +class OutOfRange(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.OUT_OF_RANGE` error.""" + + grpc_status_code = grpc.StatusCode.OUT_OF_RANGE if grpc is not None else None + + +class Unauthorized(ClientError): + """Exception mapping a ``401 Unauthorized`` response.""" + + code = http_client.UNAUTHORIZED + + +class Unauthenticated(Unauthorized): + """Exception mapping a :attr:`grpc.StatusCode.UNAUTHENTICATED` error.""" + + grpc_status_code = grpc.StatusCode.UNAUTHENTICATED if grpc is not None else None + + +class Forbidden(ClientError): + """Exception mapping a ``403 Forbidden`` response.""" + + code = http_client.FORBIDDEN + + +class PermissionDenied(Forbidden): + """Exception mapping a :attr:`grpc.StatusCode.PERMISSION_DENIED` error.""" + + grpc_status_code = grpc.StatusCode.PERMISSION_DENIED if grpc is not None else None + + +class NotFound(ClientError): + """Exception mapping a ``404 Not Found`` response or a + :attr:`grpc.StatusCode.NOT_FOUND` error.""" + + code = http_client.NOT_FOUND + grpc_status_code = grpc.StatusCode.NOT_FOUND if grpc is not None else None + + +class MethodNotAllowed(ClientError): + """Exception mapping a ``405 Method Not Allowed`` response.""" + + code = http_client.METHOD_NOT_ALLOWED + + +class Conflict(ClientError): + """Exception mapping a ``409 Conflict`` response.""" + + code = http_client.CONFLICT + + +class AlreadyExists(Conflict): + """Exception mapping a :attr:`grpc.StatusCode.ALREADY_EXISTS` error.""" + + grpc_status_code = grpc.StatusCode.ALREADY_EXISTS if grpc is not None else None + + +class Aborted(Conflict): + """Exception mapping a :attr:`grpc.StatusCode.ABORTED` error.""" + + grpc_status_code = grpc.StatusCode.ABORTED if grpc is not None else None + + +class LengthRequired(ClientError): + """Exception mapping a ``411 Length Required`` response.""" + + code = http_client.LENGTH_REQUIRED + + +class PreconditionFailed(ClientError): + """Exception mapping a ``412 Precondition Failed`` response.""" + + code = http_client.PRECONDITION_FAILED + + +class RequestRangeNotSatisfiable(ClientError): + """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" + + code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE + + +class TooManyRequests(ClientError): + """Exception mapping a ``429 Too Many Requests`` response.""" + + # http_client does not define a constant for this in Python 2. + code = 429 + + +class ResourceExhausted(TooManyRequests): + """Exception mapping a :attr:`grpc.StatusCode.RESOURCE_EXHAUSTED` error.""" + + grpc_status_code = grpc.StatusCode.RESOURCE_EXHAUSTED if grpc is not None else None + + +class Cancelled(ClientError): + """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error.""" + + # This maps to HTTP status code 499. See + # https://github.com/googleapis/googleapis/blob/master/google/rpc\ + # /code.proto + code = 499 + grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None + + +class ServerError(GoogleAPICallError): + """Base for 5xx responses.""" + + +class InternalServerError(ServerError): + """Exception mapping a ``500 Internal Server Error`` response. or a + :attr:`grpc.StatusCode.INTERNAL` error.""" + + code = http_client.INTERNAL_SERVER_ERROR + grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None + + +class Unknown(ServerError): + """Exception mapping a :attr:`grpc.StatusCode.UNKNOWN` error.""" + + grpc_status_code = grpc.StatusCode.UNKNOWN if grpc is not None else None + + +class DataLoss(ServerError): + """Exception mapping a :attr:`grpc.StatusCode.DATA_LOSS` error.""" + + grpc_status_code = grpc.StatusCode.DATA_LOSS if grpc is not None else None + + +class MethodNotImplemented(ServerError): + """Exception mapping a ``501 Not Implemented`` response or a + :attr:`grpc.StatusCode.UNIMPLEMENTED` error.""" + + code = http_client.NOT_IMPLEMENTED + grpc_status_code = grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None + + +class BadGateway(ServerError): + """Exception mapping a ``502 Bad Gateway`` response.""" + + code = http_client.BAD_GATEWAY + + +class ServiceUnavailable(ServerError): + """Exception mapping a ``503 Service Unavailable`` response or a + :attr:`grpc.StatusCode.UNAVAILABLE` error.""" + + code = http_client.SERVICE_UNAVAILABLE + grpc_status_code = grpc.StatusCode.UNAVAILABLE if grpc is not None else None + + +class GatewayTimeout(ServerError): + """Exception mapping a ``504 Gateway Timeout`` response.""" + + code = http_client.GATEWAY_TIMEOUT + + +class DeadlineExceeded(GatewayTimeout): + """Exception mapping a :attr:`grpc.StatusCode.DEADLINE_EXCEEDED` error.""" + + grpc_status_code = grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None + + +def exception_class_for_http_status(status_code): + """Return the exception class for a specific HTTP status code. + + Args: + status_code (int): The HTTP status code. + + Returns: + :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_http_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from an HTTP status code. + + Args: + status_code (int): The HTTP status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + error_class = exception_class_for_http_status(status_code) + error = error_class(message, **kwargs) + + if error.code is None: + error.code = status_code + + return error + + +def from_http_response(response): + """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`. + + Args: + response (requests.Response): The HTTP response. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`, with the message and errors populated + from the response. + """ + try: + payload = response.json() + except ValueError: + payload = {"error": {"message": response.text or "unknown error"}} + + error_message = payload.get("error", {}).get("message", "unknown error") + errors = payload.get("error", {}).get("errors", ()) + + message = "{method} {url}: {error}".format( + method=response.request.method, url=response.request.url, error=error_message + ) + + exception = from_http_status( + response.status_code, message, errors=errors, response=response + ) + return exception + + +def exception_class_for_grpc_status(status_code): + """Return the exception class for a specific :class:`grpc.StatusCode`. + + Args: + status_code (grpc.StatusCode): The gRPC status code. + + Returns: + :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _GRPC_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_grpc_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.StatusCode`. + + Args: + status_code (grpc.StatusCode): The gRPC status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + error_class = exception_class_for_grpc_status(status_code) + error = error_class(message, **kwargs) + + if error.grpc_status_code is None: + error.grpc_status_code = status_code + + return error + + +def from_grpc_error(rpc_exc): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. + + Args: + rpc_exc (grpc.RpcError): The gRPC error. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + if isinstance(rpc_exc, grpc.Call): + return from_grpc_status( + rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), response=rpc_exc + ) + else: + return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/api_core/google/api_core/future/__init__.py b/api_core/google/api_core/future/__init__.py new file mode 100644 index 000000000000..3768b2c53f53 --- /dev/null +++ b/api_core/google/api_core/future/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for dealing with asynchronous operations.""" + +from google.api_core.future.base import Future + +__all__ = ["Future"] diff --git a/api_core/google/api_core/future/_helpers.py b/api_core/google/api_core/future/_helpers.py new file mode 100644 index 000000000000..9e88ca9d561d --- /dev/null +++ b/api_core/google/api_core/future/_helpers.py @@ -0,0 +1,39 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Private helpers for futures.""" + +import logging +import threading + + +_LOGGER = logging.getLogger(__name__) + + +def start_daemon_thread(*args, **kwargs): + """Starts a thread and marks it as a daemon thread.""" + thread = threading.Thread(*args, **kwargs) + thread.daemon = True + thread.start() + return thread + + +def safe_invoke_callback(callback, *args, **kwargs): + """Invoke a callback, swallowing and logging any exceptions.""" + # pylint: disable=bare-except + # We intentionally want to swallow all exceptions. + try: + return callback(*args, **kwargs) + except Exception: + _LOGGER.exception("Error while executing Future callback.") diff --git a/api_core/google/api_core/future/base.py b/api_core/google/api_core/future/base.py new file mode 100644 index 000000000000..e7888ca3b07b --- /dev/null +++ b/api_core/google/api_core/future/base.py @@ -0,0 +1,67 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstract and helper bases for Future implementations.""" + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class Future(object): + # pylint: disable=missing-docstring + # We inherit the interfaces here from concurrent.futures. + + """Future interface. + + This interface is based on :class:`concurrent.futures.Future`. + """ + + @abc.abstractmethod + def cancel(self): + raise NotImplementedError() + + @abc.abstractmethod + def cancelled(self): + raise NotImplementedError() + + @abc.abstractmethod + def running(self): + raise NotImplementedError() + + @abc.abstractmethod + def done(self): + raise NotImplementedError() + + @abc.abstractmethod + def result(self, timeout=None): + raise NotImplementedError() + + @abc.abstractmethod + def exception(self, timeout=None): + raise NotImplementedError() + + @abc.abstractmethod + def add_done_callback(self, fn): + # pylint: disable=invalid-name + raise NotImplementedError() + + @abc.abstractmethod + def set_result(self, result): + raise NotImplementedError() + + @abc.abstractmethod + def set_exception(self, exception): + raise NotImplementedError() diff --git a/api_core/google/api_core/future/polling.py b/api_core/google/api_core/future/polling.py new file mode 100644 index 000000000000..4266e9e721e3 --- /dev/null +++ b/api_core/google/api_core/future/polling.py @@ -0,0 +1,183 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstract and helper bases for Future implementations.""" + +import abc +import concurrent.futures + +from google.api_core import exceptions +from google.api_core import retry +from google.api_core.future import _helpers +from google.api_core.future import base + + +class _OperationNotComplete(Exception): + """Private exception used for polling via retry.""" + + pass + + +RETRY_PREDICATE = retry.if_exception_type( + _OperationNotComplete, + exceptions.TooManyRequests, + exceptions.InternalServerError, + exceptions.BadGateway, +) +DEFAULT_RETRY = retry.Retry(predicate=RETRY_PREDICATE) + + +class PollingFuture(base.Future): + """A Future that needs to poll some service to check its status. + + The :meth:`done` method should be implemented by subclasses. The polling + behavior will repeatedly call ``done`` until it returns True. + + .. note: Privacy here is intended to prevent the final class from + overexposing, not to prevent subclasses from accessing methods. + + Args: + retry (google.api_core.retry.Retry): The retry configuration used + when polling. This can be used to control how often :meth:`done` + is polled. Regardless of the retry's ``deadline``, it will be + overridden by the ``timeout`` argument to :meth:`result`. + """ + + def __init__(self, retry=DEFAULT_RETRY): + super(PollingFuture, self).__init__() + self._retry = retry + self._result = None + self._exception = None + self._result_set = False + """bool: Set to True when the result has been set via set_result or + set_exception.""" + self._polling_thread = None + self._done_callbacks = [] + + @abc.abstractmethod + def done(self): + """Checks to see if the operation is complete. + + Returns: + bool: True if the operation is complete, False otherwise. + """ + # pylint: disable=redundant-returns-doc, missing-raises-doc + raise NotImplementedError() + + def _done_or_raise(self): + """Check if the future is done and raise if it's not.""" + if not self.done(): + raise _OperationNotComplete() + + def running(self): + """True if the operation is currently running.""" + return not self.done() + + def _blocking_poll(self, timeout=None): + """Poll and wait for the Future to be resolved. + + Args: + timeout (int): + How long (in seconds) to wait for the operation to complete. + If None, wait indefinitely. + """ + if self._result_set: + return + + retry_ = self._retry.with_deadline(timeout) + + try: + retry_(self._done_or_raise)() + except exceptions.RetryError: + raise concurrent.futures.TimeoutError( + "Operation did not complete within the designated " "timeout." + ) + + def result(self, timeout=None): + """Get the result of the operation, blocking if necessary. + + Args: + timeout (int): + How long (in seconds) to wait for the operation to complete. + If None, wait indefinitely. + + Returns: + google.protobuf.Message: The Operation's result. + + Raises: + google.api_core.GoogleAPICallError: If the operation errors or if + the timeout is reached before the operation completes. + """ + self._blocking_poll(timeout=timeout) + + if self._exception is not None: + # pylint: disable=raising-bad-type + # Pylint doesn't recognize that this is valid in this case. + raise self._exception + + return self._result + + def exception(self, timeout=None): + """Get the exception from the operation, blocking if necessary. + + Args: + timeout (int): How long to wait for the operation to complete. + If None, wait indefinitely. + + Returns: + Optional[google.api_core.GoogleAPICallError]: The operation's + error. + """ + self._blocking_poll(timeout=timeout) + return self._exception + + def add_done_callback(self, fn): + """Add a callback to be executed when the operation is complete. + + If the operation is not already complete, this will start a helper + thread to poll for the status of the operation in the background. + + Args: + fn (Callable[Future]): The callback to execute when the operation + is complete. + """ + if self._result_set: + _helpers.safe_invoke_callback(fn, self) + return + + self._done_callbacks.append(fn) + + if self._polling_thread is None: + # The polling thread will exit on its own as soon as the operation + # is done. + self._polling_thread = _helpers.start_daemon_thread( + target=self._blocking_poll + ) + + def _invoke_callbacks(self, *args, **kwargs): + """Invoke all done callbacks.""" + for callback in self._done_callbacks: + _helpers.safe_invoke_callback(callback, *args, **kwargs) + + def set_result(self, result): + """Set the Future's result.""" + self._result = result + self._result_set = True + self._invoke_callbacks(self) + + def set_exception(self, exception): + """Set the Future's exception.""" + self._exception = exception + self._result_set = True + self._invoke_callbacks(self) diff --git a/api_core/google/api_core/gapic_v1/__init__.py b/api_core/google/api_core/gapic_v1/__init__.py new file mode 100644 index 000000000000..e7a7a686f58e --- /dev/null +++ b/api_core/google/api_core/gapic_v1/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.api_core.gapic_v1 import client_info +from google.api_core.gapic_v1 import config +from google.api_core.gapic_v1 import method +from google.api_core.gapic_v1 import routing_header + +__all__ = ["client_info", "config", "method", "routing_header"] diff --git a/api_core/google/api_core/gapic_v1/client_info.py b/api_core/google/api_core/gapic_v1/client_info.py new file mode 100644 index 000000000000..bdc2ce440de3 --- /dev/null +++ b/api_core/google/api_core/gapic_v1/client_info.py @@ -0,0 +1,55 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for providing client information. + +Client information is used to send information about the calling client, +such as the library and Python version, to API services. +""" + +from google.api_core import client_info + + +METRICS_METADATA_KEY = "x-goog-api-client" + + +class ClientInfo(client_info.ClientInfo): + """Client information used to generate a user-agent for API calls. + + This user-agent information is sent along with API calls to allow the + receiving service to do analytics on which versions of Python and Google + libraries are being used. + + Args: + python_version (str): The Python interpreter version, for example, + ``'2.7.13'``. + grpc_version (Optional[str]): The gRPC library version. + api_core_version (str): The google-api-core library version. + gapic_version (Optional[str]): The sversion of gapic-generated client + library, if the library was generated by gapic. + client_library_version (Optional[str]): The version of the client + library, generally used if the client library was not generated + by gapic or if additional functionality was built on top of + a gapic client library. + user_agent (Optional[str]): Prefix to the user agent header. This is + used to supply information such as application name or partner tool. + Recommended format: ``application-or-tool-ID/major.minor.version``. + """ + + def to_grpc_metadata(self): + """Returns the gRPC metadata for this client info.""" + return (METRICS_METADATA_KEY, self.to_user_agent()) + + +DEFAULT_CLIENT_INFO = ClientInfo() diff --git a/api_core/google/api_core/gapic_v1/config.py b/api_core/google/api_core/gapic_v1/config.py new file mode 100644 index 000000000000..3a3eb15fdd44 --- /dev/null +++ b/api_core/google/api_core/gapic_v1/config.py @@ -0,0 +1,167 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for loading gapic configuration data. + +The Google API generator creates supplementary configuration for each RPC +method to tell the client library how to deal with retries and timeouts. +""" + +import collections + +import grpc +import six + +from google.api_core import exceptions +from google.api_core import retry +from google.api_core import timeout + + +_MILLIS_PER_SECOND = 1000.0 + + +def _exception_class_for_grpc_status_name(name): + """Returns the Google API exception class for a gRPC error code name. + + Args: + name (str): The name of the gRPC status code, for example, + ``UNAVAILABLE``. + + Returns: + :func:`type`: The appropriate subclass of + :class:`google.api_core.exceptions.GoogleAPICallError`. + """ + return exceptions.exception_class_for_grpc_status(getattr(grpc.StatusCode, name)) + + +def _retry_from_retry_config(retry_params, retry_codes): + """Creates a Retry object given a gapic retry configuration. + + Args: + retry_params (dict): The retry parameter values, for example:: + + { + "initial_retry_delay_millis": 1000, + "retry_delay_multiplier": 2.5, + "max_retry_delay_millis": 120000, + "initial_rpc_timeout_millis": 120000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 120000, + "total_timeout_millis": 600000 + } + + retry_codes (sequence[str]): The list of retryable gRPC error code + names. + + Returns: + google.api_core.retry.Retry: The default retry object for the method. + """ + exception_classes = [ + _exception_class_for_grpc_status_name(code) for code in retry_codes + ] + return retry.Retry( + retry.if_exception_type(*exception_classes), + initial=(retry_params["initial_retry_delay_millis"] / _MILLIS_PER_SECOND), + maximum=(retry_params["max_retry_delay_millis"] / _MILLIS_PER_SECOND), + multiplier=retry_params["retry_delay_multiplier"], + deadline=retry_params["total_timeout_millis"] / _MILLIS_PER_SECOND, + ) + + +def _timeout_from_retry_config(retry_params): + """Creates a ExponentialTimeout object given a gapic retry configuration. + + Args: + retry_params (dict): The retry parameter values, for example:: + + { + "initial_retry_delay_millis": 1000, + "retry_delay_multiplier": 2.5, + "max_retry_delay_millis": 120000, + "initial_rpc_timeout_millis": 120000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 120000, + "total_timeout_millis": 600000 + } + + Returns: + google.api_core.retry.ExponentialTimeout: The default time object for + the method. + """ + return timeout.ExponentialTimeout( + initial=(retry_params["initial_rpc_timeout_millis"] / _MILLIS_PER_SECOND), + maximum=(retry_params["max_rpc_timeout_millis"] / _MILLIS_PER_SECOND), + multiplier=retry_params["rpc_timeout_multiplier"], + deadline=(retry_params["total_timeout_millis"] / _MILLIS_PER_SECOND), + ) + + +MethodConfig = collections.namedtuple("MethodConfig", ["retry", "timeout"]) + + +def parse_method_configs(interface_config): + """Creates default retry and timeout objects for each method in a gapic + interface config. + + Args: + interface_config (Mapping): The interface config section of the full + gapic library config. For example, If the full configuration has + an interface named ``google.example.v1.ExampleService`` you would + pass in just that interface's configuration, for example + ``gapic_config['interfaces']['google.example.v1.ExampleService']``. + + Returns: + Mapping[str, MethodConfig]: A mapping of RPC method names to their + configuration. + """ + # Grab all the retry codes + retry_codes_map = { + name: retry_codes + for name, retry_codes in six.iteritems(interface_config.get("retry_codes", {})) + } + + # Grab all of the retry params + retry_params_map = { + name: retry_params + for name, retry_params in six.iteritems( + interface_config.get("retry_params", {}) + ) + } + + # Iterate through all the API methods and create a flat MethodConfig + # instance for each one. + method_configs = {} + + for method_name, method_params in six.iteritems( + interface_config.get("methods", {}) + ): + retry_params_name = method_params.get("retry_params_name") + + if retry_params_name is not None: + retry_params = retry_params_map[retry_params_name] + retry_ = _retry_from_retry_config( + retry_params, retry_codes_map[method_params["retry_codes_name"]] + ) + timeout_ = _timeout_from_retry_config(retry_params) + + # No retry config, so this is a non-retryable method. + else: + retry_ = None + timeout_ = timeout.ConstantTimeout( + method_params["timeout_millis"] / _MILLIS_PER_SECOND + ) + + method_configs[method_name] = MethodConfig(retry=retry_, timeout=timeout_) + + return method_configs diff --git a/api_core/google/api_core/gapic_v1/method.py b/api_core/google/api_core/gapic_v1/method.py new file mode 100644 index 000000000000..49982c0306cb --- /dev/null +++ b/api_core/google/api_core/gapic_v1/method.py @@ -0,0 +1,242 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for wrapping low-level gRPC methods with common functionality. + +This is used by gapic clients to provide common error mapping, retry, timeout, +pagination, and long-running operations to gRPC methods. +""" + +from google.api_core import general_helpers +from google.api_core import grpc_helpers +from google.api_core import timeout +from google.api_core.gapic_v1 import client_info + +USE_DEFAULT_METADATA = object() +DEFAULT = object() +"""Sentinel value indicating that a retry or timeout argument was unspecified, +so the default should be used.""" + + +def _is_not_none_or_false(value): + return value is not None and value is not False + + +def _apply_decorators(func, decorators): + """Apply a list of decorators to a given function. + + ``decorators`` may contain items that are ``None`` or ``False`` which will + be ignored. + """ + decorators = filter(_is_not_none_or_false, reversed(decorators)) + + for decorator in decorators: + func = decorator(func) + + return func + + +def _determine_timeout(default_timeout, specified_timeout, retry): + """Determines how timeout should be applied to a wrapped method. + + Args: + default_timeout (Optional[Timeout]): The default timeout specified + at method creation time. + specified_timeout (Optional[Timeout]): The timeout specified at + invocation time. If :attr:`DEFAULT`, this will be set to + the ``default_timeout``. + retry (Optional[Retry]): The retry specified at invocation time. + + Returns: + Optional[Timeout]: The timeout to apply to the method or ``None``. + """ + if specified_timeout is DEFAULT: + specified_timeout = default_timeout + + if specified_timeout is default_timeout: + # If timeout is the default and the default timeout is exponential and + # a non-default retry is specified, make sure the timeout's deadline + # matches the retry's. This handles the case where the user leaves + # the timeout default but specifies a lower deadline via the retry. + if ( + retry + and retry is not DEFAULT + and isinstance(default_timeout, timeout.ExponentialTimeout) + ): + return default_timeout.with_deadline(retry._deadline) + else: + return default_timeout + + # If timeout is specified as a number instead of a Timeout instance, + # convert it to a ConstantTimeout. + if isinstance(specified_timeout, (int, float)): + return timeout.ConstantTimeout(specified_timeout) + else: + return specified_timeout + + +class _GapicCallable(object): + """Callable that applies retry, timeout, and metadata logic. + + Args: + target (Callable): The low-level RPC method. + retry (google.api_core.retry.Retry): The default retry for the + callable. If ``None``, this callable will not retry by default + timeout (google.api_core.timeout.Timeout): The default timeout + for the callable. If ``None``, this callable will not specify + a timeout argument to the low-level RPC method by default. + metadata (Sequence[Tuple[str, str]]): Additional metadata that is + provided to the RPC method on every invocation. This is merged with + any metadata specified during invocation. If ``None``, no + additional metadata will be passed to the RPC method. + """ + + def __init__(self, target, retry, timeout, metadata=None): + self._target = target + self._retry = retry + self._timeout = timeout + self._metadata = metadata + + def __call__(self, *args, **kwargs): + """Invoke the low-level RPC with retry, timeout, and metadata.""" + # Note: Due to Python 2 lacking keyword-only arguments we use kwargs to + # extract the retry and timeout params. + timeout_ = _determine_timeout( + self._timeout, + kwargs.pop("timeout", self._timeout), + # Use only the invocation-specified retry only for this, as we only + # want to adjust the timeout deadline if the *user* specified + # a different retry. + kwargs.get("retry", None), + ) + + retry = kwargs.pop("retry", self._retry) + + if retry is DEFAULT: + retry = self._retry + + # Apply all applicable decorators. + wrapped_func = _apply_decorators(self._target, [retry, timeout_]) + + # Add the user agent metadata to the call. + if self._metadata is not None: + metadata = kwargs.get("metadata", []) + # Due to the nature of invocation, None should be treated the same + # as not specified. + if metadata is None: + metadata = [] + metadata = list(metadata) + metadata.extend(self._metadata) + kwargs["metadata"] = metadata + + return wrapped_func(*args, **kwargs) + + +def wrap_method( + func, + default_retry=None, + default_timeout=None, + client_info=client_info.DEFAULT_CLIENT_INFO, +): + """Wrap an RPC method with common behavior. + + This applies common error wrapping, retry, and timeout behavior a function. + The wrapped function will take optional ``retry`` and ``timeout`` + arguments. + + For example:: + + import google.api_core.gapic_v1.method + from google.api_core import retry + from google.api_core import timeout + + # The original RPC method. + def get_topic(name, timeout=None): + request = publisher_v2.GetTopicRequest(name=name) + return publisher_stub.GetTopic(request, timeout=timeout) + + default_retry = retry.Retry(deadline=60) + default_timeout = timeout.Timeout(deadline=60) + wrapped_get_topic = google.api_core.gapic_v1.method.wrap_method( + get_topic, default_retry) + + # Execute get_topic with default retry and timeout: + response = wrapped_get_topic() + + # Execute get_topic without doing any retying but with the default + # timeout: + response = wrapped_get_topic(retry=None) + + # Execute get_topic but only retry on 5xx errors: + my_retry = retry.Retry(retry.if_exception_type( + exceptions.InternalServerError)) + response = wrapped_get_topic(retry=my_retry) + + The way this works is by late-wrapping the given function with the retry + and timeout decorators. Essentially, when ``wrapped_get_topic()`` is + called: + + * ``get_topic()`` is first wrapped with the ``timeout`` into + ``get_topic_with_timeout``. + * ``get_topic_with_timeout`` is wrapped with the ``retry`` into + ``get_topic_with_timeout_and_retry()``. + * The final ``get_topic_with_timeout_and_retry`` is called passing through + the ``args`` and ``kwargs``. + + The callstack is therefore:: + + method.__call__() -> + Retry.__call__() -> + Timeout.__call__() -> + wrap_errors() -> + get_topic() + + Note that if ``timeout`` or ``retry`` is ``None``, then they are not + applied to the function. For example, + ``wrapped_get_topic(timeout=None, retry=None)`` is more or less + equivalent to just calling ``get_topic`` but with error re-mapping. + + Args: + func (Callable[Any]): The function to wrap. It should accept an + optional ``timeout`` argument. If ``metadata`` is not ``None``, it + should accept a ``metadata`` argument. + default_retry (Optional[google.api_core.Retry]): The default retry + strategy. If ``None``, the method will not retry by default. + default_timeout (Optional[google.api_core.Timeout]): The default + timeout strategy. Can also be specified as an int or float. If + ``None``, the method will not have timeout specified by default. + client_info + (Optional[google.api_core.gapic_v1.client_info.ClientInfo]): + Client information used to create a user-agent string that's + passed as gRPC metadata to the method. If unspecified, then + a sane default will be used. If ``None``, then no user agent + metadata will be provided to the RPC method. + + Returns: + Callable: A new callable that takes optional ``retry`` and ``timeout`` + arguments and applies the common error mapping, retry, timeout, + and metadata behavior to the low-level RPC method. + """ + func = grpc_helpers.wrap_errors(func) + + if client_info is not None: + user_agent_metadata = [client_info.to_grpc_metadata()] + else: + user_agent_metadata = None + + return general_helpers.wraps(func)( + _GapicCallable( + func, default_retry, default_timeout, metadata=user_agent_metadata + ) + ) diff --git a/api_core/google/api_core/gapic_v1/routing_header.py b/api_core/google/api_core/gapic_v1/routing_header.py new file mode 100644 index 000000000000..3fb12a6f8e26 --- /dev/null +++ b/api_core/google/api_core/gapic_v1/routing_header.py @@ -0,0 +1,62 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for constructing routing headers. + +These headers are used by Google infrastructure to determine how to route +requests, especially for services that are regional. + +Generally, these headers are specified as gRPC metadata. +""" + +import sys + +from six.moves.urllib.parse import urlencode + +ROUTING_METADATA_KEY = "x-goog-request-params" + + +def to_routing_header(params): + """Returns a routing header string for the given request parameters. + + Args: + params (Mapping[str, Any]): A dictionary containing the request + parameters used for routing. + + Returns: + str: The routing header string. + """ + if sys.version_info[0] < 3: + # Python 2 does not have the "safe" parameter for urlencode. + return urlencode(params).replace("%2F", "/") + return urlencode( + params, + # Per Google API policy (go/api-url-encoding), / is not encoded. + safe="/", + ) + + +def to_grpc_metadata(params): + """Returns the gRPC metadata containing the routing headers for the given + request parameters. + + Args: + params (Mapping[str, Any]): A dictionary containing the request + parameters used for routing. + + Returns: + Tuple(str, str): The gRPC metadata containing the routing header key + and value. + """ + return (ROUTING_METADATA_KEY, to_routing_header(params)) diff --git a/api_core/google/api_core/general_helpers.py b/api_core/google/api_core/general_helpers.py new file mode 100644 index 000000000000..d2d0c4402311 --- /dev/null +++ b/api_core/google/api_core/general_helpers.py @@ -0,0 +1,33 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for general Python functionality.""" + +import functools + +import six + + +# functools.partial objects lack several attributes present on real function +# objects. In Python 2 wraps fails on this so use a restricted set instead. +_PARTIAL_VALID_ASSIGNMENTS = ("__doc__",) + + +def wraps(wrapped): + """A functools.wraps helper that handles partial objects on Python 2.""" + # https://github.com/google/pytype/issues/322 + if isinstance(wrapped, functools.partial): # pytype: disable=wrong-arg-types + return six.wraps(wrapped, assigned=_PARTIAL_VALID_ASSIGNMENTS) + else: + return six.wraps(wrapped) diff --git a/api_core/google/api_core/grpc_helpers.py b/api_core/google/api_core/grpc_helpers.py new file mode 100644 index 000000000000..4d63beb36910 --- /dev/null +++ b/api_core/google/api_core/grpc_helpers.py @@ -0,0 +1,384 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for :mod:`grpc`.""" + +import collections + +import grpc +import six + +from google.api_core import exceptions +from google.api_core import general_helpers +import google.auth +import google.auth.credentials +import google.auth.transport.grpc +import google.auth.transport.requests + +try: + import grpc_gcp + + HAS_GRPC_GCP = True +except ImportError: + HAS_GRPC_GCP = False + +# The list of gRPC Callable interfaces that return iterators. +_STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) + + +def _patch_callable_name(callable_): + """Fix-up gRPC callable attributes. + + gRPC callable lack the ``__name__`` attribute which causes + :func:`functools.wraps` to error. This adds the attribute if needed. + """ + if not hasattr(callable_, "__name__"): + callable_.__name__ = callable_.__class__.__name__ + + +def _wrap_unary_errors(callable_): + """Map errors for Unary-Unary and Stream-Unary gRPC callables.""" + _patch_callable_name(callable_) + + @six.wraps(callable_) + def error_remapped_callable(*args, **kwargs): + try: + return callable_(*args, **kwargs) + except grpc.RpcError as exc: + six.raise_from(exceptions.from_grpc_error(exc), exc) + + return error_remapped_callable + + +class _StreamingResponseIterator(grpc.Call): + def __init__(self, wrapped): + self._wrapped = wrapped + + def __iter__(self): + """This iterator is also an iterable that returns itself.""" + return self + + def next(self): + """Get the next response from the stream. + + Returns: + protobuf.Message: A single response from the stream. + """ + try: + return six.next(self._wrapped) + except grpc.RpcError as exc: + six.raise_from(exceptions.from_grpc_error(exc), exc) + + # Alias needed for Python 2/3 support. + __next__ = next + + # grpc.Call & grpc.RpcContext interface + + def add_callback(self, callback): + return self._wrapped.add_callback(callback) + + def cancel(self): + return self._wrapped.cancel() + + def code(self): + return self._wrapped.code() + + def details(self): + return self._wrapped.details() + + def initial_metadata(self): + return self._wrapped.initial_metadata() + + def is_active(self): + return self._wrapped.is_active() + + def time_remaining(self): + return self._wrapped.time_remaining() + + def trailing_metadata(self): + return self._wrapped.trailing_metadata() + + +def _wrap_stream_errors(callable_): + """Wrap errors for Unary-Stream and Stream-Stream gRPC callables. + + The callables that return iterators require a bit more logic to re-map + errors when iterating. This wraps both the initial invocation and the + iterator of the return value to re-map errors. + """ + _patch_callable_name(callable_) + + @general_helpers.wraps(callable_) + def error_remapped_callable(*args, **kwargs): + try: + result = callable_(*args, **kwargs) + return _StreamingResponseIterator(result) + except grpc.RpcError as exc: + six.raise_from(exceptions.from_grpc_error(exc), exc) + + return error_remapped_callable + + +def wrap_errors(callable_): + """Wrap a gRPC callable and map :class:`grpc.RpcErrors` to friendly error + classes. + + Errors raised by the gRPC callable are mapped to the appropriate + :class:`google.api_core.exceptions.GoogleAPICallError` subclasses. + The original `grpc.RpcError` (which is usually also a `grpc.Call`) is + available from the ``response`` property on the mapped exception. This + is useful for extracting metadata from the original error. + + Args: + callable_ (Callable): A gRPC callable. + + Returns: + Callable: The wrapped gRPC callable. + """ + if isinstance(callable_, _STREAM_WRAP_CLASSES): + return _wrap_stream_errors(callable_) + else: + return _wrap_unary_errors(callable_) + + +def create_channel( + target, credentials=None, scopes=None, ssl_credentials=None, **kwargs +): + """Create a secure channel with credentials. + + Args: + target (str): The target service address in the format 'hostname:port'. + credentials (google.auth.credentials.Credentials): The credentials. If + not specified, then this function will attempt to ascertain the + credentials from the environment using :func:`google.auth.default`. + scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. + kwargs: Additional key-word args passed to + :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + + Returns: + grpc.Channel: The created channel. + """ + if credentials is None: + credentials, _ = google.auth.default(scopes=scopes) + else: + credentials = google.auth.credentials.with_scopes_if_required( + credentials, scopes + ) + + request = google.auth.transport.requests.Request() + + # Create the metadata plugin for inserting the authorization header. + metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( + credentials, request + ) + + # Create a set of grpc.CallCredentials using the metadata plugin. + google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin) + + if ssl_credentials is None: + ssl_credentials = grpc.ssl_channel_credentials() + + # Combine the ssl credentials and the authorization credentials. + composite_credentials = grpc.composite_channel_credentials( + ssl_credentials, google_auth_credentials + ) + + if HAS_GRPC_GCP: + # If grpc_gcp module is available use grpc_gcp.secure_channel, + # otherwise, use grpc.secure_channel to create grpc channel. + return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) + else: + return grpc.secure_channel(target, composite_credentials, **kwargs) + + +_MethodCall = collections.namedtuple( + "_MethodCall", ("request", "timeout", "metadata", "credentials") +) + +_ChannelRequest = collections.namedtuple("_ChannelRequest", ("method", "request")) + + +class _CallableStub(object): + """Stub for the grpc.*MultiCallable interfaces.""" + + def __init__(self, method, channel): + self._method = method + self._channel = channel + self.response = None + """Union[protobuf.Message, Callable[protobuf.Message], exception]: + The response to give when invoking this callable. If this is a + callable, it will be invoked with the request protobuf. If it's an + exception, the exception will be raised when this is invoked. + """ + self.responses = None + """Iterator[ + Union[protobuf.Message, Callable[protobuf.Message], exception]]: + An iterator of responses. If specified, self.response will be populated + on each invocation by calling ``next(self.responses)``.""" + self.requests = [] + """List[protobuf.Message]: All requests sent to this callable.""" + self.calls = [] + """List[Tuple]: All invocations of this callable. Each tuple is the + request, timeout, metadata, and credentials.""" + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + self._channel.requests.append(_ChannelRequest(self._method, request)) + self.calls.append(_MethodCall(request, timeout, metadata, credentials)) + self.requests.append(request) + + response = self.response + if self.responses is not None: + if response is None: + response = next(self.responses) + else: + raise ValueError( + "{method}.response and {method}.responses are mutually " + "exclusive.".format(method=self._method) + ) + + if callable(response): + return response(request) + + if isinstance(response, Exception): + raise response + + if response is not None: + return response + + raise ValueError('Method stub for "{}" has no response.'.format(self._method)) + + +def _simplify_method_name(method): + """Simplifies a gRPC method name. + + When gRPC invokes the channel to create a callable, it gives a full + method name like "/google.pubsub.v1.Publisher/CreateTopic". This + returns just the name of the method, in this case "CreateTopic". + + Args: + method (str): The name of the method. + + Returns: + str: The simplified name of the method. + """ + return method.rsplit("/", 1).pop() + + +class ChannelStub(grpc.Channel): + """A testing stub for the grpc.Channel interface. + + This can be used to test any client that eventually uses a gRPC channel + to communicate. By passing in a channel stub, you can configure which + responses are returned and track which requests are made. + + For example: + + .. code-block:: python + + channel_stub = grpc_helpers.ChannelStub() + client = FooClient(channel=channel_stub) + + channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') + + foo = client.get_foo(labels=['baz']) + + assert foo.name == 'bar' + assert channel_stub.GetFoo.requests[0].labels = ['baz'] + + Each method on the stub can be accessed and configured on the channel. + Here's some examples of various configurations: + + .. code-block:: python + + # Return a basic response: + + channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') + assert client.get_foo().name == 'bar' + + # Raise an exception: + channel_stub.GetFoo.response = NotFound('...') + + with pytest.raises(NotFound): + client.get_foo() + + # Use a sequence of responses: + channel_stub.GetFoo.responses = iter([ + foo_pb2.Foo(name='bar'), + foo_pb2.Foo(name='baz'), + ]) + + assert client.get_foo().name == 'bar' + assert client.get_foo().name == 'baz' + + # Use a callable + + def on_get_foo(request): + return foo_pb2.Foo(name='bar' + request.id) + + channel_stub.GetFoo.response = on_get_foo + + assert client.get_foo(id='123').name == 'bar123' + """ + + def __init__(self, responses=[]): + self.requests = [] + """Sequence[Tuple[str, protobuf.Message]]: A list of all requests made + on this channel in order. The tuple is of method name, request + message.""" + self._method_stubs = {} + + def _stub_for_method(self, method): + method = _simplify_method_name(method) + self._method_stubs[method] = _CallableStub(method, self) + return self._method_stubs[method] + + def __getattr__(self, key): + try: + return self._method_stubs[key] + except KeyError: + raise AttributeError + + def unary_unary(self, method, request_serializer=None, response_deserializer=None): + """grpc.Channel.unary_unary implementation.""" + return self._stub_for_method(method) + + def unary_stream(self, method, request_serializer=None, response_deserializer=None): + """grpc.Channel.unary_stream implementation.""" + return self._stub_for_method(method) + + def stream_unary(self, method, request_serializer=None, response_deserializer=None): + """grpc.Channel.stream_unary implementation.""" + return self._stub_for_method(method) + + def stream_stream( + self, method, request_serializer=None, response_deserializer=None + ): + """grpc.Channel.stream_stream implementation.""" + return self._stub_for_method(method) + + def subscribe(self, callback, try_to_connect=False): + """grpc.Channel.subscribe implementation.""" + pass + + def unsubscribe(self, callback): + """grpc.Channel.unsubscribe implementation.""" + pass + + def close(self): + """grpc.Channel.close implementation.""" + pass diff --git a/api_core/google/api_core/iam.py b/api_core/google/api_core/iam.py new file mode 100644 index 000000000000..0e108a30679a --- /dev/null +++ b/api_core/google/api_core/iam.py @@ -0,0 +1,273 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Non-API-specific IAM policy definitions + +For allowed roles / permissions, see: +https://cloud.google.com/iam/docs/understanding-roles + +Example usage: + +.. code-block:: python + + # ``get_iam_policy`` returns a :class:'~google.api_core.iam.Policy`. + policy = resource.get_iam_policy() + + phred = policy.user("phred@example.com") + admin_group = policy.group("admins@groups.example.com") + account = policy.service_account("account-1234@accounts.example.com") + policy["roles/owner"] = [phred, admin_group, account] + policy["roles/editor"] = policy.authenticated_users() + policy["roles/viewer"] = policy.all_users() + + resource.set_iam_policy(policy) +""" + +import collections +try: + from collections import abc as collections_abc +except ImportError: # Python 2.7 + import collections as collections_abc +import warnings + +# Generic IAM roles + +OWNER_ROLE = "roles/owner" +"""Generic role implying all rights to an object.""" + +EDITOR_ROLE = "roles/editor" +"""Generic role implying rights to modify an object.""" + +VIEWER_ROLE = "roles/viewer" +"""Generic role implying rights to access an object.""" + +_ASSIGNMENT_DEPRECATED_MSG = """\ +Assigning to '{}' is deprecated. Replace with 'policy[{}] = members.""" + + +class Policy(collections_abc.MutableMapping): + """IAM Policy + + See + https://cloud.google.com/iam/reference/rest/v1/Policy + + Args: + etag (Optional[str]): ETag used to identify a unique of the policy + version (Optional[int]): unique version of the policy + """ + + _OWNER_ROLES = (OWNER_ROLE,) + """Roles mapped onto our ``owners`` attribute.""" + + _EDITOR_ROLES = (EDITOR_ROLE,) + """Roles mapped onto our ``editors`` attribute.""" + + _VIEWER_ROLES = (VIEWER_ROLE,) + """Roles mapped onto our ``viewers`` attribute.""" + + def __init__(self, etag=None, version=None): + self.etag = etag + self.version = version + self._bindings = collections.defaultdict(set) + + def __iter__(self): + return iter(self._bindings) + + def __len__(self): + return len(self._bindings) + + def __getitem__(self, key): + return self._bindings[key] + + def __setitem__(self, key, value): + self._bindings[key] = set(value) + + def __delitem__(self, key): + del self._bindings[key] + + @property + def owners(self): + """Legacy access to owner role. + + DEPRECATED: use ``policy["roles/owners"]`` instead.""" + result = set() + for role in self._OWNER_ROLES: + for member in self._bindings.get(role, ()): + result.add(member) + return frozenset(result) + + @owners.setter + def owners(self, value): + """Update owners. + + DEPRECATED: use ``policy["roles/owners"] = value`` instead.""" + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("owners", OWNER_ROLE), DeprecationWarning + ) + self[OWNER_ROLE] = value + + @property + def editors(self): + """Legacy access to editor role. + + DEPRECATED: use ``policy["roles/editors"]`` instead.""" + result = set() + for role in self._EDITOR_ROLES: + for member in self._bindings.get(role, ()): + result.add(member) + return frozenset(result) + + @editors.setter + def editors(self, value): + """Update editors. + + DEPRECATED: use ``policy["roles/editors"] = value`` instead.""" + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("editors", EDITOR_ROLE), + DeprecationWarning, + ) + self[EDITOR_ROLE] = value + + @property + def viewers(self): + """Legacy access to viewer role. + + DEPRECATED: use ``policy["roles/viewers"]`` instead + """ + result = set() + for role in self._VIEWER_ROLES: + for member in self._bindings.get(role, ()): + result.add(member) + return frozenset(result) + + @viewers.setter + def viewers(self, value): + """Update viewers. + + DEPRECATED: use ``policy["roles/viewers"] = value`` instead. + """ + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("viewers", VIEWER_ROLE), + DeprecationWarning, + ) + self[VIEWER_ROLE] = value + + @staticmethod + def user(email): + """Factory method for a user member. + + Args: + email (str): E-mail for this particular user. + + Returns: + str: A member string corresponding to the given user. + """ + return "user:%s" % (email,) + + @staticmethod + def service_account(email): + """Factory method for a service account member. + + Args: + email (str): E-mail for this particular service account. + + Returns: + str: A member string corresponding to the given service account. + """ + return "serviceAccount:%s" % (email,) + + @staticmethod + def group(email): + """Factory method for a group member. + + Args: + email (str): An id or e-mail for this particular group. + + Returns: + str: A member string corresponding to the given group. + """ + return "group:%s" % (email,) + + @staticmethod + def domain(domain): + """Factory method for a domain member. + + Args: + domain (str): The domain for this member. + + Returns: + str: A member string corresponding to the given domain. + """ + return "domain:%s" % (domain,) + + @staticmethod + def all_users(): + """Factory method for a member representing all users. + + Returns: + str: A member string representing all users. + """ + return "allUsers" + + @staticmethod + def authenticated_users(): + """Factory method for a member representing all authenticated users. + + Returns: + str: A member string representing all authenticated users. + """ + return "allAuthenticatedUsers" + + @classmethod + def from_api_repr(cls, resource): + """Factory: create a policy from a JSON resource. + + Args: + resource (dict): policy resource returned by ``getIamPolicy`` API. + + Returns: + :class:`Policy`: the parsed policy + """ + version = resource.get("version") + etag = resource.get("etag") + policy = cls(etag, version) + for binding in resource.get("bindings", ()): + role = binding["role"] + members = sorted(binding["members"]) + policy[role] = members + return policy + + def to_api_repr(self): + """Render a JSON policy resource. + + Returns: + dict: a resource to be passed to the ``setIamPolicy`` API. + """ + resource = {} + + if self.etag is not None: + resource["etag"] = self.etag + + if self.version is not None: + resource["version"] = self.version + + if self._bindings: + bindings = resource["bindings"] = [] + for role, members in sorted(self._bindings.items()): + if members: + bindings.append({"role": role, "members": sorted(set(members))}) + + if not bindings: + del resource["bindings"] + + return resource diff --git a/api_core/google/api_core/operation.py b/api_core/google/api_core/operation.py new file mode 100644 index 000000000000..87f42a973e1b --- /dev/null +++ b/api_core/google/api_core/operation.py @@ -0,0 +1,320 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operation using :meth:`Operation.result`: + + +.. code-block:: python + + operation = my_api_client.long_running_method() + result = operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + operation = my_api_client.long_running_method() + + def my_callback(future): + result = future.result() + + operation.add_done_callback(my_callback) + +""" + +import functools +import threading + +from google.api_core import exceptions +from google.api_core import protobuf_helpers +from google.api_core.future import polling +from google.longrunning import operations_pb2 +from google.protobuf import json_format +from google.rpc import code_pb2 + + +class Operation(polling.PollingFuture): + """A Future for interacting with a Google API Long-Running Operation. + + Args: + operation (google.longrunning.operations_pb2.Operation): The + initial operation. + refresh (Callable[[], ~.api_core.operation.Operation]): A callable that + returns the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel + the operation. + result_type (func:`type`): The protobuf type for the operation's + result. + metadata_type (func:`type`): The protobuf type for the operation's + metadata. + retry (google.api_core.retry.Retry): The retry configuration used + when polling. This can be used to control how often :meth:`done` + is polled. Regardless of the retry's ``deadline``, it will be + overridden by the ``timeout`` argument to :meth:`result`. + """ + + def __init__( + self, + operation, + refresh, + cancel, + result_type, + metadata_type=None, + retry=polling.DEFAULT_RETRY, + ): + super(Operation, self).__init__(retry=retry) + self._operation = operation + self._refresh = refresh + self._cancel = cancel + self._result_type = result_type + self._metadata_type = metadata_type + self._completion_lock = threading.Lock() + # Invoke this in case the operation came back already complete. + self._set_result_from_operation() + + @property + def operation(self): + """google.longrunning.Operation: The current long-running operation.""" + return self._operation + + @property + def metadata(self): + """google.protobuf.Message: the current operation metadata.""" + if not self._operation.HasField("metadata"): + return None + + return protobuf_helpers.from_any_pb( + self._metadata_type, self._operation.metadata + ) + + @classmethod + def deserialize(self, payload): + """Deserialize a ``google.longrunning.Operation`` protocol buffer. + + Args: + payload (bytes): A serialized operation protocol buffer. + + Returns: + ~.operations_pb2.Operation: An Operation protobuf object. + """ + return operations_pb2.Operation.FromString(payload) + + def _set_result_from_operation(self): + """Set the result or exception from the operation if it is complete.""" + # This must be done in a lock to prevent the polling thread + # and main thread from both executing the completion logic + # at the same time. + with self._completion_lock: + # If the operation isn't complete or if the result has already been + # set, do not call set_result/set_exception again. + # Note: self._result_set is set to True in set_result and + # set_exception, in case those methods are invoked directly. + if not self._operation.done or self._result_set: + return + + if self._operation.HasField("response"): + response = protobuf_helpers.from_any_pb( + self._result_type, self._operation.response + ) + self.set_result(response) + elif self._operation.HasField("error"): + exception = exceptions.GoogleAPICallError( + self._operation.error.message, + errors=(self._operation.error,), + response=self._operation, + ) + self.set_exception(exception) + else: + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) + + def _refresh_and_update(self): + """Refresh the operation and update the result if needed.""" + # If the currently cached operation is done, no need to make another + # RPC as it will not change once done. + if not self._operation.done: + self._operation = self._refresh() + self._set_result_from_operation() + + def done(self): + """Checks to see if the operation is complete. + + Returns: + bool: True if the operation is complete, False otherwise. + """ + self._refresh_and_update() + return self._operation.done + + def cancel(self): + """Attempt to cancel the operation. + + Returns: + bool: True if the cancel RPC was made, False if the operation is + already complete. + """ + if self.done(): + return False + + self._cancel() + return True + + def cancelled(self): + """True if the operation was cancelled.""" + self._refresh_and_update() + return ( + self._operation.HasField("error") + and self._operation.error.code == code_pb2.CANCELLED + ) + + +def _refresh_http(api_request, operation_name): + """Refresh an operation using a JSON/HTTP client. + + Args: + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + operation_name (str): The name of the operation. + + Returns: + google.longrunning.operations_pb2.Operation: The operation. + """ + path = "operations/{}".format(operation_name) + api_response = api_request(method="GET", path=path) + return json_format.ParseDict(api_response, operations_pb2.Operation()) + + +def _cancel_http(api_request, operation_name): + """Cancel an operation using a JSON/HTTP client. + + Args: + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + operation_name (str): The name of the operation. + """ + path = "operations/{}:cancel".format(operation_name) + api_request(method="POST", path=path) + + +def from_http_json(operation, api_request, result_type, **kwargs): + """Create an operation future using a HTTP/JSON client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via `HTTP/JSON`_. + + .. _HTTP/JSON: https://cloud.google.com/speech/reference/rest/\ + v1beta1/operations#Operation + + Args: + operation (dict): Operation as a dictionary. + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + result_type (:func:`type`): The protobuf result type. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + operation_proto = json_format.ParseDict(operation, operations_pb2.Operation()) + refresh = functools.partial(_refresh_http, api_request, operation_proto.name) + cancel = functools.partial(_cancel_http, api_request, operation_proto.name) + return Operation(operation_proto, refresh, cancel, result_type, **kwargs) + + +def _refresh_grpc(operations_stub, operation_name): + """Refresh an operation using a gRPC client. + + Args: + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The gRPC operations stub. + operation_name (str): The name of the operation. + + Returns: + google.longrunning.operations_pb2.Operation: The operation. + """ + request_pb = operations_pb2.GetOperationRequest(name=operation_name) + return operations_stub.GetOperation(request_pb) + + +def _cancel_grpc(operations_stub, operation_name): + """Cancel an operation using a gRPC client. + + Args: + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The gRPC operations stub. + operation_name (str): The name of the operation. + """ + request_pb = operations_pb2.CancelOperationRequest(name=operation_name) + operations_stub.CancelOperation(request_pb) + + +def from_grpc(operation, operations_stub, result_type, **kwargs): + """Create an operation future using a gRPC client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via gRPC. + + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + + Args: + operation (google.longrunning.operations_pb2.Operation): The operation. + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The operations stub. + result_type (:func:`type`): The protobuf result type. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + refresh = functools.partial(_refresh_grpc, operations_stub, operation.name) + cancel = functools.partial(_cancel_grpc, operations_stub, operation.name) + return Operation(operation, refresh, cancel, result_type, **kwargs) + + +def from_gapic(operation, operations_client, result_type, **kwargs): + """Create an operation future from a gapic client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via a gapic client. + + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + + Args: + operation (google.longrunning.operations_pb2.Operation): The operation. + operations_client (google.api_core.operations_v1.OperationsClient): + The operations client. + result_type (:func:`type`): The protobuf result type. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + refresh = functools.partial(operations_client.get_operation, operation.name) + cancel = functools.partial(operations_client.cancel_operation, operation.name) + return Operation(operation, refresh, cancel, result_type, **kwargs) diff --git a/api_core/google/api_core/operations_v1/__init__.py b/api_core/google/api_core/operations_v1/__init__.py new file mode 100644 index 000000000000..f0549561b6ff --- /dev/null +++ b/api_core/google/api_core/operations_v1/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Package for interacting with the google.longrunning.operations meta-API.""" + +from google.api_core.operations_v1.operations_client import OperationsClient + +__all__ = ["OperationsClient"] diff --git a/api_core/google/api_core/operations_v1/operations_client.py b/api_core/google/api_core/operations_v1/operations_client.py new file mode 100644 index 000000000000..cd2923bb9d09 --- /dev/null +++ b/api_core/google/api_core/operations_v1/operations_client.py @@ -0,0 +1,288 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A client for the google.longrunning.operations meta-API. + +This is a client that deals with long-running operations that follow the +pattern outlined by the `Google API Style Guide`_. + +When an API method normally takes long time to complete, it can be designed to +return ``Operation`` to the client, and the client can use this interface to +receive the real response asynchronously by polling the operation resource to +receive the response. + +It is not a separate service, but rather an interface implemented by a larger +service. The protocol-level definition is available at +`google/longrunning/operations.proto`_. Typically, this will be constructed +automatically by another client class to deal with operations. + +.. _Google API Style Guide: + https://cloud.google.com/apis/design/design_pattern + s#long_running_operations +.. _google/longrunning/operations.proto: + https://github.com/googleapis/googleapis/blob/master/google/longrunning + /operations.proto +""" + +import functools + +from google.api_core import gapic_v1 +from google.api_core import page_iterator +from google.api_core.operations_v1 import operations_client_config +from google.longrunning import operations_pb2 + + +class OperationsClient(object): + """Client for interacting with long-running operations within a service. + + Args: + channel (grpc.Channel): The gRPC channel associated with the service + that implements the ``google.longrunning.operations`` interface. + client_config (dict): + A dictionary of call options for each method. If not specified + the default configuration is used. + """ + + def __init__(self, channel, client_config=operations_client_config.config): + # Create the gRPC client stub. + self.operations_stub = operations_pb2.OperationsStub(channel) + + # Create all wrapped methods using the interface configuration. + # The interface config contains all of the default settings for retry + # and timeout for each RPC method. + interfaces = client_config["interfaces"] + interface_config = interfaces["google.longrunning.Operations"] + method_configs = gapic_v1.config.parse_method_configs(interface_config) + + self._get_operation = gapic_v1.method.wrap_method( + self.operations_stub.GetOperation, + default_retry=method_configs["GetOperation"].retry, + default_timeout=method_configs["GetOperation"].timeout, + ) + + self._list_operations = gapic_v1.method.wrap_method( + self.operations_stub.ListOperations, + default_retry=method_configs["ListOperations"].retry, + default_timeout=method_configs["ListOperations"].timeout, + ) + + self._cancel_operation = gapic_v1.method.wrap_method( + self.operations_stub.CancelOperation, + default_retry=method_configs["CancelOperation"].retry, + default_timeout=method_configs["CancelOperation"].timeout, + ) + + self._delete_operation = gapic_v1.method.wrap_method( + self.operations_stub.DeleteOperation, + default_retry=method_configs["DeleteOperation"].retry, + default_timeout=method_configs["DeleteOperation"].timeout, + ) + + # Service calls + def get_operation( + self, name, retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT + ): + """Gets the latest state of a long-running operation. + + Clients can use this method to poll the operation result at intervals + as recommended by the API service. + + Example: + >>> from google.api_core import operations_v1 + >>> api = operations_v1.OperationsClient() + >>> name = '' + >>> response = api.get_operation(name) + + Args: + name (str): The name of the operation resource. + retry (google.api_core.retry.Retry): The retry strategy to use + when invoking the RPC. If unspecified, the default retry from + the client configuration will be used. If ``None``, then this + method will not retry the RPC at all. + timeout (float): The amount of time in seconds to wait for the RPC + to complete. Note that if ``retry`` is used, this timeout + applies to each individual attempt and the overall time it + takes for this method to complete may be longer. If + unspecified, the the default timeout in the client + configuration is used. If ``None``, then the RPC method will + not time out. + + Returns: + google.longrunning.operations_pb2.Operation: The state of the + operation. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If an error occurred + while invoking the RPC, the appropriate ``GoogleAPICallError`` + subclass will be raised. + """ + request = operations_pb2.GetOperationRequest(name=name) + return self._get_operation(request, retry=retry, timeout=timeout) + + def list_operations( + self, + name, + filter_, + retry=gapic_v1.method.DEFAULT, + timeout=gapic_v1.method.DEFAULT, + ): + """ + Lists operations that match the specified filter in the request. + + Example: + >>> from google.api_core import operations_v1 + >>> api = operations_v1.OperationsClient() + >>> name = '' + >>> + >>> # Iterate over all results + >>> for operation in api.list_operations(name): + >>> # process operation + >>> pass + >>> + >>> # Or iterate over results one page at a time + >>> iter = api.list_operations(name) + >>> for page in iter.pages: + >>> for operation in page: + >>> # process operation + >>> pass + + Args: + name (str): The name of the operation collection. + filter_ (str): The standard list filter. + retry (google.api_core.retry.Retry): The retry strategy to use + when invoking the RPC. If unspecified, the default retry from + the client configuration will be used. If ``None``, then this + method will not retry the RPC at all. + timeout (float): The amount of time in seconds to wait for the RPC + to complete. Note that if ``retry`` is used, this timeout + applies to each individual attempt and the overall time it + takes for this method to complete may be longer. If + unspecified, the the default timeout in the client + configuration is used. If ``None``, then the RPC method will + not time out. + + Returns: + google.api_core.page_iterator.Iterator: An iterator that yields + :class:`google.longrunning.operations_pb2.Operation` instances. + + Raises: + google.api_core.exceptions.MethodNotImplemented: If the server + does not support this method. Services are not required to + implement this method. + google.api_core.exceptions.GoogleAPICallError: If an error occurred + while invoking the RPC, the appropriate ``GoogleAPICallError`` + subclass will be raised. + """ + # Create the request object. + request = operations_pb2.ListOperationsRequest(name=name, filter=filter_) + + # Create the method used to fetch pages + method = functools.partial(self._list_operations, retry=retry, timeout=timeout) + + iterator = page_iterator.GRPCIterator( + client=None, + method=method, + request=request, + items_field="operations", + request_token_field="page_token", + response_token_field="next_page_token", + ) + + return iterator + + def cancel_operation( + self, name, retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT + ): + """Starts asynchronous cancellation on a long-running operation. + + The server makes a best effort to cancel the operation, but success is + not guaranteed. Clients can use :meth:`get_operation` or service- + specific methods to check whether the cancellation succeeded or whether + the operation completed despite cancellation. On successful + cancellation, the operation is not deleted; instead, it becomes an + operation with an ``Operation.error`` value with a + ``google.rpc.Status.code`` of ``1``, corresponding to + ``Code.CANCELLED``. + + Example: + >>> from google.api_core import operations_v1 + >>> api = operations_v1.OperationsClient() + >>> name = '' + >>> api.cancel_operation(name) + + Args: + name (str): The name of the operation resource to be cancelled. + retry (google.api_core.retry.Retry): The retry strategy to use + when invoking the RPC. If unspecified, the default retry from + the client configuration will be used. If ``None``, then this + method will not retry the RPC at all. + timeout (float): The amount of time in seconds to wait for the RPC + to complete. Note that if ``retry`` is used, this timeout + applies to each individual attempt and the overall time it + takes for this method to complete may be longer. If + unspecified, the the default timeout in the client + configuration is used. If ``None``, then the RPC method will + not time out. + + Raises: + google.api_core.exceptions.MethodNotImplemented: If the server + does not support this method. Services are not required to + implement this method. + google.api_core.exceptions.GoogleAPICallError: If an error occurred + while invoking the RPC, the appropriate ``GoogleAPICallError`` + subclass will be raised. + """ + # Create the request object. + request = operations_pb2.CancelOperationRequest(name=name) + self._cancel_operation(request, retry=retry, timeout=timeout) + + def delete_operation( + self, name, retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT + ): + """Deletes a long-running operation. + + This method indicates that the client is no longer interested in the + operation result. It does not cancel the operation. + + Example: + >>> from google.api_core import operations_v1 + >>> api = operations_v1.OperationsClient() + >>> name = '' + >>> api.delete_operation(name) + + Args: + name (str): The name of the operation resource to be deleted. + retry (google.api_core.retry.Retry): The retry strategy to use + when invoking the RPC. If unspecified, the default retry from + the client configuration will be used. If ``None``, then this + method will not retry the RPC at all. + timeout (float): The amount of time in seconds to wait for the RPC + to complete. Note that if ``retry`` is used, this timeout + applies to each individual attempt and the overall time it + takes for this method to complete may be longer. If + unspecified, the the default timeout in the client + configuration is used. If ``None``, then the RPC method will + not time out. + + Raises: + google.api_core.exceptions.MethodNotImplemented: If the server + does not support this method. Services are not required to + implement this method. + google.api_core.exceptions.GoogleAPICallError: If an error occurred + while invoking the RPC, the appropriate ``GoogleAPICallError`` + subclass will be raised. + """ + # Create the request object. + request = operations_pb2.DeleteOperationRequest(name=name) + self._delete_operation(request, retry=retry, timeout=timeout) diff --git a/api_core/google/api_core/operations_v1/operations_client_config.py b/api_core/google/api_core/operations_v1/operations_client_config.py new file mode 100644 index 000000000000..6cf95753f4be --- /dev/null +++ b/api_core/google/api_core/operations_v1/operations_client_config.py @@ -0,0 +1,59 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""gapic configuration for the googe.longrunning.operations client.""" + +config = { + "interfaces": { + "google.longrunning.Operations": { + "retry_codes": { + "idempotent": ["DEADLINE_EXCEEDED", "UNAVAILABLE"], + "non_idempotent": [], + }, + "retry_params": { + "default": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 20000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 600000, + "total_timeout_millis": 600000, + } + }, + "methods": { + "GetOperation": { + "timeout_millis": 60000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + "ListOperations": { + "timeout_millis": 60000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + "CancelOperation": { + "timeout_millis": 60000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + "DeleteOperation": { + "timeout_millis": 60000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + }, + } + } +} diff --git a/api_core/google/api_core/page_iterator.py b/api_core/google/api_core/page_iterator.py new file mode 100644 index 000000000000..3ac5904399b0 --- /dev/null +++ b/api_core/google/api_core/page_iterator.py @@ -0,0 +1,549 @@ +# Copyright 2015 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Iterators for paging through paged API methods. + +These iterators simplify the process of paging through API responses +where the request takes a page token and the response is a list of results with +a token for the next page. See `list pagination`_ in the Google API Style Guide +for more details. + +.. _list pagination: + https://cloud.google.com/apis/design/design_patterns#list_pagination + +API clients that have methods that follow the list pagination pattern can +return an :class:`.Iterator`. You can use this iterator to get **all** of +the results across all pages:: + + >>> results_iterator = client.list_resources() + >>> list(results_iterator) # Convert to a list (consumes all values). + +Or you can walk your way through items and call off the search early if +you find what you're looking for (resulting in possibly fewer requests):: + + >>> for resource in results_iterator: + ... print(resource.name) + ... if not resource.is_valid: + ... break + +At any point, you may check the number of items consumed by referencing the +``num_results`` property of the iterator:: + + >>> for my_item in results_iterator: + ... if results_iterator.num_results >= 10: + ... break + +When iterating, not every new item will send a request to the server. +To iterate based on each page of items (where a page corresponds to +a request):: + + >>> for page in results_iterator.pages: + ... print('=' * 20) + ... print(' Page number: {:d}'.format(iterator.page_number)) + ... print(' Items in page: {:d}'.format(page.num_items)) + ... print(' First item: {!r}'.format(next(page))) + ... print('Items remaining: {:d}'.format(page.remaining)) + ... print('Next page token: {}'.format(iterator.next_page_token)) + ==================== + Page number: 1 + Items in page: 1 + First item: + Items remaining: 0 + Next page token: eav1OzQB0OM8rLdGXOEsyQWSG + ==================== + Page number: 2 + Items in page: 19 + First item: + Items remaining: 18 + Next page token: None + +Then, for each page you can get all the resources on that page by iterating +through it or using :func:`list`:: + + >>> list(page) + [ + , + , + , + ] +""" + +import abc + +import six + + +class Page(object): + """Single page of results in an iterator. + + Args: + parent (google.api_core.page_iterator.Iterator): The iterator that owns + the current page. + items (Sequence[Any]): An iterable (that also defines __len__) of items + from a raw API response. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the raw API response + into the native object. Will be called with the iterator and a + single item. + """ + + def __init__(self, parent, items, item_to_value): + self._parent = parent + self._num_items = len(items) + self._remaining = self._num_items + self._item_iter = iter(items) + self._item_to_value = item_to_value + + @property + def num_items(self): + """int: Total items in the page.""" + return self._num_items + + @property + def remaining(self): + """int: Remaining items in the page.""" + return self._remaining + + def __iter__(self): + """The :class:`Page` is an iterator of items.""" + return self + + def next(self): + """Get the next value in the page.""" + item = six.next(self._item_iter) + result = self._item_to_value(self._parent, item) + # Since we've successfully got the next value from the + # iterator, we update the number of remaining. + self._remaining -= 1 + return result + + # Alias needed for Python 2/3 support. + __next__ = next + + +def _item_to_value_identity(iterator, item): + """An item to value transformer that returns the item un-changed.""" + # pylint: disable=unused-argument + # We are conforming to the interface defined by Iterator. + return item + + +@six.add_metaclass(abc.ABCMeta) +class Iterator(object): + """A generic class for iterating through API list responses. + + Args: + client(google.cloud.client.Client): The API client. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the raw API response + into the native object. Will be called with the iterator and a + single item. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. + """ + + def __init__( + self, + client, + item_to_value=_item_to_value_identity, + page_token=None, + max_results=None, + ): + self._started = False + self.client = client + """Optional[Any]: The client that created this iterator.""" + self.item_to_value = item_to_value + """Callable[Iterator, Any]: Callable to convert an item from the type + in the raw API response into the native object. Will be called with + the iterator and a + single item. + """ + self.max_results = max_results + """int: The maximum number of results to fetch.""" + + # The attributes below will change over the life of the iterator. + self.page_number = 0 + """int: The current page of results.""" + self.next_page_token = page_token + """str: The token for the next page of results. If this is set before + the iterator starts, it effectively offsets the iterator to a + specific starting point.""" + self.num_results = 0 + """int: The total number of results fetched so far.""" + + @property + def pages(self): + """Iterator of pages in the response. + + returns: + types.GeneratorType[google.api_core.page_iterator.Page]: A + generator of page instances. + + raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._page_iter(increment=True) + + def _items_iter(self): + """Iterator for each item returned.""" + for page in self._page_iter(increment=False): + for item in page: + self.num_results += 1 + yield item + + def __iter__(self): + """Iterator for each item returned. + + Returns: + types.GeneratorType[Any]: A generator of items from the API. + + Raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._items_iter() + + def _page_iter(self, increment): + """Generator of pages of API responses. + + Args: + increment (bool): Flag indicating if the total number of results + should be incremented on each page. This is useful since a page + iterator will want to increment by results per page while an + items iterator will want to increment per item. + + Yields: + Page: each page of items from the API. + """ + page = self._next_page() + while page is not None: + self.page_number += 1 + if increment: + self.num_results += page.num_items + yield page + page = self._next_page() + + @abc.abstractmethod + def _next_page(self): + """Get the next page in the iterator. + + This does nothing and is intended to be over-ridden by subclasses + to return the next :class:`Page`. + + Raises: + NotImplementedError: Always, this method is abstract. + """ + raise NotImplementedError + + +def _do_nothing_page_start(iterator, page, response): + """Helper to provide custom behavior after a :class:`Page` is started. + + This is a do-nothing stand-in as the default value. + + Args: + iterator (Iterator): An iterator that holds some request info. + page (Page): The page that was just created. + response (Any): The API response for a page. + """ + # pylint: disable=unused-argument + pass + + +class HTTPIterator(Iterator): + """A generic class for iterating through HTTP/JSON API list responses. + + To make an iterator work, you'll need to provide a way to convert a JSON + item returned from the API into the object of your choice (via + ``item_to_value``). You also may need to specify a custom ``items_key`` so + that a given response (containing a page of results) can be parsed into an + iterable page of the actual objects you want. + + Args: + client (google.cloud.client.Client): The API client. + api_request (Callable): The function to use to make API requests. + Generally, this will be + :meth:`google.cloud._http.JSONConnection.api_request`. + path (str): The method path to query for the list of items. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the JSON response into + a native object. Will be called with the iterator and a single + item. + items_key (str): The key in the API response where the list of items + can be found. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. + extra_params (dict): Extra query string parameters for the + API call. + page_start (Callable[ + google.api_core.page_iterator.Iterator, + google.api_core.page_iterator.Page, dict]): Callable to provide + any special behavior after a new page has been created. Assumed + signature takes the :class:`.Iterator` that started the page, + the :class:`.Page` that was started and the dictionary containing + the page response. + next_token (str): The name of the field used in the response for page + tokens. + + .. autoattribute:: pages + """ + + _DEFAULT_ITEMS_KEY = "items" + _PAGE_TOKEN = "pageToken" + _MAX_RESULTS = "maxResults" + _NEXT_TOKEN = "nextPageToken" + _RESERVED_PARAMS = frozenset([_PAGE_TOKEN]) + _HTTP_METHOD = "GET" + + def __init__( + self, + client, + api_request, + path, + item_to_value, + items_key=_DEFAULT_ITEMS_KEY, + page_token=None, + max_results=None, + extra_params=None, + page_start=_do_nothing_page_start, + next_token=_NEXT_TOKEN, + ): + super(HTTPIterator, self).__init__( + client, item_to_value, page_token=page_token, max_results=max_results + ) + self.api_request = api_request + self.path = path + self._items_key = items_key + self.extra_params = extra_params + self._page_start = page_start + self._next_token = next_token + # Verify inputs / provide defaults. + if self.extra_params is None: + self.extra_params = {} + self._verify_params() + + def _verify_params(self): + """Verifies the parameters don't use any reserved parameter. + + Raises: + ValueError: If a reserved parameter is used. + """ + reserved_in_use = self._RESERVED_PARAMS.intersection(self.extra_params) + if reserved_in_use: + raise ValueError("Using a reserved parameter", reserved_in_use) + + def _next_page(self): + """Get the next page in the iterator. + + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. + """ + if self._has_next_page(): + response = self._get_next_page_response() + items = response.get(self._items_key, ()) + page = Page(self, items, self.item_to_value) + self._page_start(self, page, response) + self.next_page_token = response.get(self._next_token) + return page + else: + return None + + def _has_next_page(self): + """Determines whether or not there are more pages with results. + + Returns: + bool: Whether the iterator has more pages. + """ + if self.page_number == 0: + return True + + if self.max_results is not None: + if self.num_results >= self.max_results: + return False + + return self.next_page_token is not None + + def _get_query_params(self): + """Getter for query parameters for the next request. + + Returns: + dict: A dictionary of query parameters. + """ + result = {} + if self.next_page_token is not None: + result[self._PAGE_TOKEN] = self.next_page_token + if self.max_results is not None: + result[self._MAX_RESULTS] = self.max_results - self.num_results + result.update(self.extra_params) + return result + + def _get_next_page_response(self): + """Requests the next page from the path provided. + + Returns: + dict: The parsed JSON response of the next page's contents. + + Raises: + ValueError: If the HTTP method is not ``GET`` or ``POST``. + """ + params = self._get_query_params() + if self._HTTP_METHOD == "GET": + return self.api_request( + method=self._HTTP_METHOD, path=self.path, query_params=params + ) + elif self._HTTP_METHOD == "POST": + return self.api_request( + method=self._HTTP_METHOD, path=self.path, data=params + ) + else: + raise ValueError("Unexpected HTTP method", self._HTTP_METHOD) + + +class _GAXIterator(Iterator): + """A generic class for iterating through Cloud gRPC APIs list responses. + + Any: + client (google.cloud.client.Client): The API client. + page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped + to conform to the :class:`Iterator` interface. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the the protobuf response into a native object. Will + be called with the iterator and a single item. + max_results (int): The maximum number of results to fetch. + + .. autoattribute:: pages + """ + + def __init__(self, client, page_iter, item_to_value, max_results=None): + super(_GAXIterator, self).__init__( + client, + item_to_value, + page_token=page_iter.page_token, + max_results=max_results, + ) + self._gax_page_iter = page_iter + + def _next_page(self): + """Get the next page in the iterator. + + Wraps the response from the :class:`~google.gax.PageIterator` in a + :class:`Page` instance and captures some state at each page. + + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. + """ + try: + items = six.next(self._gax_page_iter) + page = Page(self, items, self.item_to_value) + self.next_page_token = self._gax_page_iter.page_token or None + return page + except StopIteration: + return None + + +class GRPCIterator(Iterator): + """A generic class for iterating through gRPC list responses. + + .. note:: The class does not take a ``page_token`` argument because it can + just be specified in the ``request``. + + Args: + client (google.cloud.client.Client): The API client. This unused by + this class, but kept to satisfy the :class:`Iterator` interface. + method (Callable[protobuf.Message]): A bound gRPC method that should + take a single message for the request. + request (protobuf.Message): The request message. + items_field (str): The field in the response message that has the + items for the page. + item_to_value (Callable[GRPCIterator, Any]): Callable to convert an + item from the type in the JSON response into a native object. Will + be called with the iterator and a single item. + request_token_field (str): The field in the request message used to + specify the page token. + response_token_field (str): The field in the response message that has + the token for the next page. + max_results (int): The maximum number of results to fetch. + + .. autoattribute:: pages + """ + + _DEFAULT_REQUEST_TOKEN_FIELD = "page_token" + _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token" + + def __init__( + self, + client, + method, + request, + items_field, + item_to_value=_item_to_value_identity, + request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD, + response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD, + max_results=None, + ): + super(GRPCIterator, self).__init__( + client, item_to_value, max_results=max_results + ) + self._method = method + self._request = request + self._items_field = items_field + self._request_token_field = request_token_field + self._response_token_field = response_token_field + + def _next_page(self): + """Get the next page in the iterator. + + Returns: + Page: The next page in the iterator or :data:`None` if + there are no pages left. + """ + if not self._has_next_page(): + return None + + if self.next_page_token is not None: + setattr(self._request, self._request_token_field, self.next_page_token) + + response = self._method(self._request) + + self.next_page_token = getattr(response, self._response_token_field) + items = getattr(response, self._items_field) + page = Page(self, items, self.item_to_value) + + return page + + def _has_next_page(self): + """Determines whether or not there are more pages with results. + + Returns: + bool: Whether the iterator has more pages. + """ + if self.page_number == 0: + return True + + if self.max_results is not None: + if self.num_results >= self.max_results: + return False + + # Note: intentionally a falsy check instead of a None check. The RPC + # can return an empty string indicating no more pages. + return True if self.next_page_token else False diff --git a/api_core/google/api_core/path_template.py b/api_core/google/api_core/path_template.py new file mode 100644 index 000000000000..bb549356d284 --- /dev/null +++ b/api_core/google/api_core/path_template.py @@ -0,0 +1,197 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expand and validate URL path templates. + +This module provides the :func:`expand` and :func:`validate` functions for +interacting with Google-style URL `path templates`_ which are commonly used +in Google APIs for `resource names`_. + +.. _path templates: https://github.com/googleapis/googleapis/blob + /57e2d376ac7ef48681554204a3ba78a414f2c533/google/api/http.proto#L212 +.. _resource names: https://cloud.google.com/apis/design/resource_names +""" + +from __future__ import unicode_literals + +import functools +import re + +import six + +# Regular expression for extracting variable parts from a path template. +# The variables can be expressed as: +# +# - "*": a single-segment positional variable, for example: "books/*" +# - "**": a multi-segment positional variable, for example: "shelf/**/book/*" +# - "{name}": a single-segment wildcard named variable, for example +# "books/{name}" +# - "{name=*}: same as above. +# - "{name=**}": a multi-segment wildcard named variable, for example +# "shelf/{name=**}" +# - "{name=/path/*/**}": a multi-segment named variable with a sub-template. +_VARIABLE_RE = re.compile( + r""" + ( # Capture the entire variable expression + (?P\*\*?) # Match & capture * and ** positional variables. + | + # Match & capture named variables {name} + { + (?P[^/]+?) + # Optionally match and capture the named variable's template. + (?:=(?P