Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2174800

Browse files
authored
docs: Move the schedule_export samples from python-docs-samples (#344)
Moved fromhttps://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/datastore/schedule-export
1 parent4095ce8 commit2174800

File tree

7 files changed

+491
-0
lines changed

7 files changed

+491
-0
lines changed
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#Scheduling Datastore exports with Cloud Functions and Cloud Scheduler
2+
3+
This sample application demonstrates how to schedule exports of your Datastore entities. To deploy this sample, see:
4+
5+
[Scheduling exports](https://cloud.google.com/datastore/docs/schedule-export)
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright 2021 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
importbase64
16+
importjson
17+
importos
18+
19+
fromgoogle.cloudimportdatastore_admin_v1
20+
21+
project_id=os.environ.get("GCP_PROJECT")
22+
client=datastore_admin_v1.DatastoreAdminClient()
23+
24+
25+
defdatastore_export(event,context):
26+
"""Triggers a Datastore export from a Cloud Scheduler job.
27+
28+
Args:
29+
event (dict): event[data] must contain a json object encoded in
30+
base-64. Cloud Scheduler encodes payloads in base-64 by default.
31+
Object must include a 'bucket' value and can include 'kinds'
32+
and 'namespaceIds' values.
33+
context (google.cloud.functions.Context): The Cloud Functions event
34+
metadata.
35+
"""
36+
if"data"inevent:
37+
# Triggered via Cloud Scheduler, decode the inner data field of the json payload.
38+
json_data=json.loads(base64.b64decode(event["data"]).decode("utf-8"))
39+
else:
40+
# Otherwise, for instance if triggered via the Cloud Console on a Cloud Function, the event is the data.
41+
json_data=event
42+
43+
bucket=json_data["bucket"]
44+
entity_filter=datastore_admin_v1.EntityFilter()
45+
46+
if"kinds"injson_data:
47+
entity_filter.kinds=json_data["kinds"]
48+
49+
if"namespaceIds"injson_data:
50+
entity_filter.namespace_ids=json_data["namespaceIds"]
51+
52+
export_request=datastore_admin_v1.ExportEntitiesRequest(
53+
project_id=project_id,output_url_prefix=bucket,entity_filter=entity_filter
54+
)
55+
operation=client.export_entities(request=export_request)
56+
response=operation.result()
57+
print(response)
Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,312 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__importprint_function
16+
17+
importglob
18+
importos
19+
frompathlibimportPath
20+
importsys
21+
fromtypingimportCallable,Dict,List,Optional
22+
23+
importnox
24+
25+
26+
# WARNING - WARNING - WARNING - WARNING - WARNING
27+
# WARNING - WARNING - WARNING - WARNING - WARNING
28+
# DO NOT EDIT THIS FILE EVER!
29+
# WARNING - WARNING - WARNING - WARNING - WARNING
30+
# WARNING - WARNING - WARNING - WARNING - WARNING
31+
32+
BLACK_VERSION="black==22.3.0"
33+
ISORT_VERSION="isort==5.10.1"
34+
35+
# Copy `noxfile_config.py` to your directory and modify it instead.
36+
37+
# `TEST_CONFIG` dict is a configuration hook that allows users to
38+
# modify the test configurations. The values here should be in sync
39+
# with `noxfile_config.py`. Users will copy `noxfile_config.py` into
40+
# their directory and modify it.
41+
42+
TEST_CONFIG= {
43+
# You can opt out from the test for specific Python versions.
44+
"ignored_versions": [],
45+
# Old samples are opted out of enforcing Python type hints
46+
# All new samples should feature them
47+
"enforce_type_hints":False,
48+
# An envvar key for determining the project id to use. Change it
49+
# to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
50+
# build specific Cloud project. You can also use your own string
51+
# to use your own Cloud project.
52+
"gcloud_project_env":"GOOGLE_CLOUD_PROJECT",
53+
# 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
54+
# If you need to use a specific version of pip,
55+
# change pip_version_override to the string representation
56+
# of the version number, for example, "20.2.4"
57+
"pip_version_override":None,
58+
# A dictionary you want to inject into your test. Don't put any
59+
# secrets here. These values will override predefined values.
60+
"envs": {},
61+
}
62+
63+
64+
try:
65+
# Ensure we can import noxfile_config in the project's directory.
66+
sys.path.append(".")
67+
fromnoxfile_configimportTEST_CONFIG_OVERRIDE
68+
exceptImportErrorase:
69+
print("No user noxfile_config found: detail: {}".format(e))
70+
TEST_CONFIG_OVERRIDE= {}
71+
72+
# Update the TEST_CONFIG with the user supplied values.
73+
TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)
74+
75+
76+
defget_pytest_env_vars()->Dict[str,str]:
77+
"""Returns a dict for pytest invocation."""
78+
ret= {}
79+
80+
# Override the GCLOUD_PROJECT and the alias.
81+
env_key=TEST_CONFIG["gcloud_project_env"]
82+
# This should error out if not set.
83+
ret["GOOGLE_CLOUD_PROJECT"]=os.environ[env_key]
84+
85+
# Apply user supplied envs.
86+
ret.update(TEST_CONFIG["envs"])
87+
returnret
88+
89+
90+
# DO NOT EDIT - automatically generated.
91+
# All versions used to test samples.
92+
ALL_VERSIONS= ["3.7","3.8","3.9","3.10"]
93+
94+
# Any default versions that should be ignored.
95+
IGNORED_VERSIONS=TEST_CONFIG["ignored_versions"]
96+
97+
TESTED_VERSIONS=sorted([vforvinALL_VERSIONSifvnotinIGNORED_VERSIONS])
98+
99+
INSTALL_LIBRARY_FROM_SOURCE=os.environ.get("INSTALL_LIBRARY_FROM_SOURCE",False)in (
100+
"True",
101+
"true",
102+
)
103+
104+
# Error if a python version is missing
105+
nox.options.error_on_missing_interpreters=True
106+
107+
#
108+
# Style Checks
109+
#
110+
111+
112+
def_determine_local_import_names(start_dir:str)->List[str]:
113+
"""Determines all import names that should be considered "local".
114+
115+
This is used when running the linter to insure that import order is
116+
properly checked.
117+
"""
118+
file_ext_pairs= [os.path.splitext(path)forpathinos.listdir(start_dir)]
119+
return [
120+
basename
121+
forbasename,extensioninfile_ext_pairs
122+
ifextension==".py"
123+
oros.path.isdir(os.path.join(start_dir,basename))
124+
andbasenamenotin ("__pycache__")
125+
]
126+
127+
128+
# Linting with flake8.
129+
#
130+
# We ignore the following rules:
131+
# E203: whitespace before ‘:’
132+
# E266: too many leading ‘#’ for block comment
133+
# E501: line too long
134+
# I202: Additional newline in a section of imports
135+
#
136+
# We also need to specify the rules which are ignored by default:
137+
# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121']
138+
FLAKE8_COMMON_ARGS= [
139+
"--show-source",
140+
"--builtin=gettext",
141+
"--max-complexity=20",
142+
"--import-order-style=google",
143+
"--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py",
144+
"--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202",
145+
"--max-line-length=88",
146+
]
147+
148+
149+
@nox.session
150+
deflint(session:nox.sessions.Session)->None:
151+
ifnotTEST_CONFIG["enforce_type_hints"]:
152+
session.install("flake8","flake8-import-order")
153+
else:
154+
session.install("flake8","flake8-import-order","flake8-annotations")
155+
156+
local_names=_determine_local_import_names(".")
157+
args=FLAKE8_COMMON_ARGS+ [
158+
"--application-import-names",
159+
",".join(local_names),
160+
".",
161+
]
162+
session.run("flake8",*args)
163+
164+
165+
#
166+
# Black
167+
#
168+
169+
170+
@nox.session
171+
defblacken(session:nox.sessions.Session)->None:
172+
"""Run black. Format code to uniform standard."""
173+
session.install(BLACK_VERSION)
174+
python_files= [pathforpathinos.listdir(".")ifpath.endswith(".py")]
175+
176+
session.run("black",*python_files)
177+
178+
179+
#
180+
# format = isort + black
181+
#
182+
183+
@nox.session
184+
defformat(session:nox.sessions.Session)->None:
185+
"""
186+
Run isort to sort imports. Then run black
187+
to format code to uniform standard.
188+
"""
189+
session.install(BLACK_VERSION,ISORT_VERSION)
190+
python_files= [pathforpathinos.listdir(".")ifpath.endswith(".py")]
191+
192+
# Use the --fss option to sort imports using strict alphabetical order.
193+
# See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections
194+
session.run("isort","--fss",*python_files)
195+
session.run("black",*python_files)
196+
197+
198+
#
199+
# Sample Tests
200+
#
201+
202+
203+
PYTEST_COMMON_ARGS= ["--junitxml=sponge_log.xml"]
204+
205+
206+
def_session_tests(
207+
session:nox.sessions.Session,post_install:Callable=None
208+
)->None:
209+
# check for presence of tests
210+
test_list=glob.glob("*_test.py")+glob.glob("test_*.py")
211+
test_list.extend(glob.glob("tests"))
212+
213+
iflen(test_list)==0:
214+
print("No tests found, skipping directory.")
215+
return
216+
217+
ifTEST_CONFIG["pip_version_override"]:
218+
pip_version=TEST_CONFIG["pip_version_override"]
219+
session.install(f"pip=={pip_version}")
220+
"""Runs py.test for a particular project."""
221+
concurrent_args= []
222+
ifos.path.exists("requirements.txt"):
223+
ifos.path.exists("constraints.txt"):
224+
session.install("-r","requirements.txt","-c","constraints.txt")
225+
else:
226+
session.install("-r","requirements.txt")
227+
withopen("requirements.txt")asrfile:
228+
packages=rfile.read()
229+
230+
ifos.path.exists("requirements-test.txt"):
231+
ifos.path.exists("constraints-test.txt"):
232+
session.install(
233+
"-r","requirements-test.txt","-c","constraints-test.txt"
234+
)
235+
else:
236+
session.install("-r","requirements-test.txt")
237+
withopen("requirements-test.txt")asrtfile:
238+
packages+=rtfile.read()
239+
240+
ifINSTALL_LIBRARY_FROM_SOURCE:
241+
session.install("-e",_get_repo_root())
242+
243+
ifpost_install:
244+
post_install(session)
245+
246+
if"pytest-parallel"inpackages:
247+
concurrent_args.extend(['--workers','auto','--tests-per-worker','auto'])
248+
elif"pytest-xdist"inpackages:
249+
concurrent_args.extend(['-n','auto'])
250+
251+
session.run(
252+
"pytest",
253+
*(PYTEST_COMMON_ARGS+session.posargs+concurrent_args),
254+
# Pytest will return 5 when no tests are collected. This can happen
255+
# on travis where slow and flaky tests are excluded.
256+
# See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
257+
success_codes=[0,5],
258+
env=get_pytest_env_vars(),
259+
)
260+
261+
262+
@nox.session(python=ALL_VERSIONS)
263+
defpy(session:nox.sessions.Session)->None:
264+
"""Runs py.test for a sample using the specified version of Python."""
265+
ifsession.pythoninTESTED_VERSIONS:
266+
_session_tests(session)
267+
else:
268+
session.skip(
269+
"SKIPPED: {} tests are disabled for this sample.".format(session.python)
270+
)
271+
272+
273+
#
274+
# Readmegen
275+
#
276+
277+
278+
def_get_repo_root()->Optional[str]:
279+
""" Returns the root folder of the project. """
280+
# Get root of this repository. Assume we don't have directories nested deeper than 10 items.
281+
p=Path(os.getcwd())
282+
foriinrange(10):
283+
ifpisNone:
284+
break
285+
ifPath(p/".git").exists():
286+
returnstr(p)
287+
# .git is not available in repos cloned via Cloud Build
288+
# setup.py is always in the library's root, so use that instead
289+
# https://github.com/googleapis/synthtool/issues/792
290+
ifPath(p/"setup.py").exists():
291+
returnstr(p)
292+
p=p.parent
293+
raiseException("Unable to detect repository root.")
294+
295+
296+
GENERATED_READMES=sorted([xforxinPath(".").rglob("*.rst.in")])
297+
298+
299+
@nox.session
300+
@nox.parametrize("path",GENERATED_READMES)
301+
defreadmegen(session:nox.sessions.Session,path:str)->None:
302+
"""(Re-)generates the readme for a sample."""
303+
session.install("jinja2","pyyaml")
304+
dir_=os.path.dirname(path)
305+
306+
ifos.path.exists(os.path.join(dir_,"requirements.txt")):
307+
session.install("-r",os.path.join(dir_,"requirements.txt"))
308+
309+
in_file=os.path.join(dir_,"README.rst.in")
310+
session.run(
311+
"python",_get_repo_root()+"/scripts/readme-gen/readme_gen.py",in_file
312+
)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp