Visitar URL original
feat: Introduce proper py_library and py_test support in bazel rules … · googleapis/google-cloud-python@12ee205 · GitHub
Skip to content

Commit 12ee205

Browse files
vam-googlevam-google
authored
feat: Introduce proper py_library and py_test support in bazel rules (#1360)
These changes allow running the generated python gapic unit tests on the fly by bazel right after generation in our CI. As a result of this change some of the issues with the generated code became immediatelly visible (because now we ran the generated tests) and will be addressed separately in these issues: googleapis/gapic-generator-python#1358 googleapis/gapic-generator-python#1359 The tests can be now added in BUILD.bazel files as follows: ```bzl py_test( name = "credentials_py_gapic_test", srcs = [ "credentials_py_gapic_pytest.py", "credentials_py_gapic_test.py", ], legacy_create_init = False, deps = [ ":credentials_py_gapic", ], ) ```
1 parent 115f679 commit 12ee205

9 files changed

Lines changed: 159 additions & 9 deletions

File tree

‎packages/gapic-generator/.github/workflows/tests.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,8 @@ jobs:
311311
echo "and it will start over with a clean cache."
312312
echo "The old one will disappear after 7 days."
313313
- name: Integration Tests
314-
run: bazel test tests/integration:asset tests/integration:credentials tests/integration:eventarc tests/integration:logging tests/integration:redis
314+
run: bazel test //tests/integration/...
315+
315316
goldens-lint:
316317
runs-on: ubuntu-latest
317318
steps:

‎packages/gapic-generator/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ PyYAML==6.0
99
dataclasses==0.6 # TODO(busunkim) remove when 3.6 support is dropped
1010
setuptools==63.2.0
1111
grpc-google-iam-v1==0.12.4
12+
proto-plus==1.20.6
13+
pytest-asyncio==0.19.0

‎packages/gapic-generator/rules_python_gapic/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
exports_files(["test.py", "pytest.py"])

‎packages/gapic-generator/rules_python_gapic/py_gapic.bzl‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,27 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
load("@rules_gapic//:gapic.bzl", "proto_custom_library")
15+
load("@rules_gapic//:gapic.bzl", "proto_custom_library", "unzipped_srcjar")
16+
load("@rules_python//python:defs.bzl", "py_library")
17+
load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement")
18+
19+
def _gapic_test_file_impl(ctx):
20+
generated_test_file = ctx.actions.declare_file(ctx.label.name)
21+
22+
ctx.actions.expand_template(
23+
template = ctx.attr.template.files.to_list()[0],
24+
output = generated_test_file,
25+
substitutions = {},
26+
)
27+
28+
return [DefaultInfo(files = depset(direct = [generated_test_file]))]
29+
30+
gapic_test_file = rule(
31+
_gapic_test_file_impl,
32+
attrs = {
33+
"template": attr.label(allow_files = True),
34+
},
35+
)
1636

1737
def py_gapic_library(
1838
name,
@@ -22,9 +42,9 @@ def py_gapic_library(
2242
opt_args = None,
2343
metadata = True,
2444
service_yaml = None,
45+
deps = [],
2546
**kwargs):
26-
# srcjar_target_name = "%s_srcjar" % name
27-
srcjar_target_name = name
47+
srcjar_target_name = "%s_srcjar" % name
2848
srcjar_output_suffix = ".srcjar"
2949

3050
plugin_args = plugin_args or []
@@ -50,3 +70,40 @@ def py_gapic_library(
5070
output_suffix = srcjar_output_suffix,
5171
**kwargs
5272
)
73+
74+
main_file = "%s" % srcjar_target_name + srcjar_output_suffix
75+
main_dir = "%s.py" % srcjar_target_name
76+
77+
unzipped_srcjar(
78+
name = main_dir,
79+
srcjar = ":%s" % main_file,
80+
)
81+
82+
actual_deps = deps + [
83+
"@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio",
84+
requirement("protobuf"),
85+
requirement("proto-plus"),
86+
requirement("google-api-core"),
87+
requirement("googleapis-common-protos"),
88+
requirement("pytest-asyncio"),
89+
]
90+
91+
py_library(
92+
name = name,
93+
srcs = [":%s" % main_dir],
94+
deps = actual_deps,
95+
)
96+
97+
test_file_target_name = "%s_test.py" % name
98+
99+
gapic_test_file(
100+
name = test_file_target_name,
101+
template = Label("//rules_python_gapic:test.py"),
102+
)
103+
104+
test_runner_file_target_name = "%s_pytest.py" % name
105+
106+
gapic_test_file(
107+
name = test_runner_file_target_name,
108+
template = Label("//rules_python_gapic:pytest.py"),
109+
)

‎packages/gapic-generator/rules_python_gapic/py_gapic_pkg.bzl‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,12 @@ load("@rules_gapic//:gapic_pkg.bzl", "construct_package_dir_paths")
1616

1717
def _py_gapic_src_pkg_impl(ctx):
1818
srcjar_srcs = []
19+
dir_srcs = []
1920
for dep in ctx.attr.deps:
2021
for f in dep.files.to_list():
21-
if f.extension in ("srcjar", "jar", "zip"):
22+
if f.is_directory:
23+
dir_srcs.append(f)
24+
elif f.extension in ("srcjar", "jar", "zip"):
2225
srcjar_srcs.append(f)
2326

2427
paths = construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name)
@@ -28,21 +31,27 @@ def _py_gapic_src_pkg_impl(ctx):
2831
for srcjar_src in {srcjar_srcs}; do
2932
unzip -q -o $srcjar_src -d {package_dir_path}
3033
done
34+
for dir_src in {dir_srcs}; do
35+
cp -rT -L $dir_src {package_dir_path}
36+
done
37+
# Replace 555 (forced by Bazel) permissions with 644
38+
find {package_dir_path} -type f -exec chmod 644 {{}} \\;
3139
cd {package_dir_path}/..
3240
tar -zchpf {package_dir}/{package_dir}.tar.gz {package_dir}
3341
cd -
3442
mv {package_dir_path}/{package_dir}.tar.gz {pkg}
3543
rm -rf {package_dir_path}
3644
""".format(
3745
srcjar_srcs = " ".join(["'%s'" % f.path for f in srcjar_srcs]),
46+
dir_srcs = " ".join(["'%s'" % f.path for f in dir_srcs]),
3847
package_dir_path = paths.package_dir_path,
3948
package_dir = paths.package_dir,
4049
pkg = ctx.outputs.pkg.path,
4150
package_dir_expr = paths.package_dir_expr,
4251
)
4352

4453
ctx.actions.run_shell(
45-
inputs = srcjar_srcs,
54+
inputs = srcjar_srcs + dir_srcs,
4655
command = script,
4756
outputs = [ctx.outputs.pkg],
4857
)

‎packages/gapic-generator/rules_python_gapic/pytest.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import sys
2+
import pytest
3+
import os
4+
5+
6+
if __name__ == '__main__':
7+
sys.exit(pytest.main([
8+
'--disable-pytest-warnings',
9+
'--quiet',
10+
os.path.dirname(os.path.abspath(__file__))
11+
]))

‎packages/gapic-generator/rules_python_gapic/test.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import os
2+
import sys
3+
4+
5+
if __name__ == '__main__':
6+
os.environ['PYTHONNOUSERSITE'] = 'True'
7+
entry_point_script = os.path.join(
8+
os.path.dirname(os.path.abspath(__file__)),
9+
os.path.basename(__file__).replace("_test.py", "_pytest.py"))
10+
args = [sys.executable, entry_point_script] + sys.argv[1:]
11+
os.execv(args[0], args)

‎packages/gapic-generator/rules_python_gapic/test/integration_test.bzl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def _diff_integration_goldens_impl(ctx):
1212

1313
script = """
1414
mkdir codegen_tmp
15-
unzip {input_srcs} -d codegen_tmp
15+
cp -rT -L {input_srcs} codegen_tmp
1616
diff -r codegen_tmp $PWD/tests/integration/goldens/{api_name} > {diff_output}
1717
exit 0 # Avoid a build failure.
1818
""".format(
@@ -94,7 +94,7 @@ def _overwrite_golden_impl(ctx):
9494

9595
script = """
9696
mkdir codegen_tmp
97-
unzip {input_srcs} -d codegen_tmp
97+
cp -rT -L {input_srcs} codegen_tmp
9898
cd codegen_tmp
9999
zip -r ../{goldens_output_zip} .
100100
""".format(

‎packages/gapic-generator/tests/integration/BUILD.bazel‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ package(default_visibility = ["//visibility:public"])
2626
INTEGRATION_TEST_LIBRARIES = [
2727
"asset", # Basic case.
2828
"credentials", # Check that the capital name edge case is handled.
29-
"eventarc", # create_channel is a reserved term in transport layer.
29+
"eventarc", # create_channel is a reserved term in transport layer.
3030
"logging", # Java package remapping in gapic.yaml.
3131
"redis", # Has a gapic.yaml.
3232
]
@@ -67,6 +67,18 @@ py_gapic_library(
6767
],
6868
)
6969

70+
py_test(
71+
name = "credentials_py_gapic_test",
72+
srcs = [
73+
"credentials_py_gapic_pytest.py",
74+
"credentials_py_gapic_test.py",
75+
],
76+
legacy_create_init = False,
77+
deps = [
78+
":credentials_py_gapic",
79+
],
80+
)
81+
7082
# Eventarc.
7183
py_gapic_library(
7284
name = "eventarc_py_gapic",
@@ -79,6 +91,19 @@ py_gapic_library(
7991
],
8092
)
8193

94+
# Uncomment once https://github.com/googleapis/gapic-generator-python/issues/1358 is fixed
95+
#py_test(
96+
# name = "eventarc_py_gapic_test",
97+
# srcs = [
98+
# "eventarc_py_gapic_pytest.py",
99+
# "eventarc_py_gapic_test.py",
100+
# ],
101+
# legacy_create_init = False,
102+
# deps = [
103+
# ":eventarc_py_gapic",
104+
# ],
105+
#)
106+
82107
# Logging.
83108
py_gapic_library(
84109
name = "logging_py_gapic",
@@ -91,6 +116,19 @@ py_gapic_library(
91116
],
92117
)
93118

119+
# Uncomment once https://github.com/googleapis/gapic-generator-python/issues/1359 is fixed
120+
#py_test(
121+
# name = "logging_py_gapic_test",
122+
# srcs = [
123+
# "logging_py_gapic_pytest.py",
124+
# "logging_py_gapic_test.py",
125+
# ],
126+
# legacy_create_init = False,
127+
# deps = [
128+
# ":logging_py_gapic",
129+
# ],
130+
#)
131+
94132
py_gapic_library(
95133
name = "redis_py_gapic",
96134
srcs = ["@com_google_googleapis//google/cloud/redis/v1:redis_proto"],
@@ -99,3 +137,23 @@ py_gapic_library(
99137
"autogen-snippets",
100138
],
101139
)
140+
141+
py_test(
142+
name = "redis_py_gapic_test",
143+
srcs = [
144+
"redis_py_gapic_pytest.py",
145+
"redis_py_gapic_test.py",
146+
],
147+
legacy_create_init = False,
148+
deps = [
149+
":redis_py_gapic",
150+
],
151+
)
152+
153+
test_suite(
154+
name = "googleapis_test_suite",
155+
tests = [
156+
":credentials_py_gapic_test",
157+
":redis_py_gapic_test",
158+
],
159+
)

0 commit comments

Comments
 (0)