Visitar URL original
feat: add a CLI tool to validate generation configuration (#2691) · googleapis/google-cloud-java@f52f8cc · GitHub
Skip to content

Commit f52f8cc

Browse files
JoeWang1127JoeWang1127
authored
feat: add a CLI tool to validate generation configuration (#2691)
In this PR: - Add a CLI tool to validate generation config. Downstream libraries, e.g., google-cloud-java, can write a workflow job like: ``` docker run ... python /src/cli/entry_point.py \ validate-generation-config \ --generation-config-path=path/to/generation_config.yaml ```
1 parent cb5ada0 commit f52f8cc

10 files changed

Lines changed: 315 additions & 83 deletions

‎sdk-platform-java/library_generation/cli/entry_point.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
import os
15+
import sys
1516

1617
import click as click
1718
from library_generation.generate_pr_description import generate_pr_descriptions
@@ -142,5 +143,28 @@ def generate(
142143
)
143144

144145

146+
@main.command()
147+
@click.option(
148+
"--generation-config-path",
149+
required=False,
150+
type=str,
151+
help="""
152+
Absolute or relative path to a generation_config.yaml.
153+
Default to generation_config.yaml in the current working directory.
154+
""",
155+
)
156+
def validate_generation_config(generation_config_path: str) -> None:
157+
"""
158+
Validate the given generation configuration.
159+
"""
160+
if generation_config_path is None:
161+
generation_config_path = "generation_config.yaml"
162+
try:
163+
from_yaml(os.path.abspath(generation_config_path))
164+
except ValueError as err:
165+
print(err)
166+
sys.exit(1)
167+
168+
145169
if __name__ == "__main__":
146170
main()

‎sdk-platform-java/library_generation/model/generation_config.py‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
15-
16-
1715
import yaml
18-
from typing import List, Optional, Dict
16+
from typing import Optional
1917
from library_generation.model.library_config import LibraryConfig
2018
from library_generation.model.gapic_config import GapicConfig
2119

20+
REPO_LEVEL_PARAMETER = "Repo level parameter"
21+
LIBRARY_LEVEL_PARAMETER = "Library level parameter"
22+
GAPIC_LEVEL_PARAMETER = "GAPIC level parameter"
23+
2224

2325
class GenerationConfig:
2426
"""
@@ -32,8 +34,8 @@ def __init__(
3234
libraries_bom_version: str,
3335
owlbot_cli_image: str,
3436
synthtool_commitish: str,
35-
template_excludes: List[str],
36-
libraries: List[LibraryConfig],
37+
template_excludes: list[str],
38+
libraries: list[LibraryConfig],
3739
grpc_version: Optional[str] = None,
3840
protoc_version: Optional[str] = None,
3941
):
@@ -46,6 +48,7 @@ def __init__(
4648
self.libraries = libraries
4749
self.grpc_version = grpc_version
4850
self.protoc_version = protoc_version
51+
self.__validate()
4952

5053
def get_proto_path_to_library_name(self) -> dict[str, str]:
5154
"""
@@ -62,6 +65,19 @@ def get_proto_path_to_library_name(self) -> dict[str, str]:
6265
def is_monorepo(self) -> bool:
6366
return len(self.libraries) > 1
6467

68+
def __validate(self) -> None:
69+
seen_library_names = dict()
70+
for library in self.libraries:
71+
library_name = library.get_library_name()
72+
if library_name in seen_library_names:
73+
raise ValueError(
74+
f"Both {library.name_pretty} and "
75+
f"{seen_library_names.get(library_name)} have the same "
76+
f"library name: {library_name}, please update one of the "
77+
f"library to have a different library name."
78+
)
79+
seen_library_names[library_name] = library.name_pretty
80+
6581

6682
def from_yaml(path_to_yaml: str) -> GenerationConfig:
6783
"""
@@ -72,15 +88,19 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
7288
with open(path_to_yaml, "r") as file_stream:
7389
config = yaml.safe_load(file_stream)
7490

75-
libraries = __required(config, "libraries")
91+
libraries = __required(config, "libraries", REPO_LEVEL_PARAMETER)
92+
if not libraries:
93+
raise ValueError(f"Library is None in {path_to_yaml}.")
7694

7795
parsed_libraries = list()
7896
for library in libraries:
7997
gapics = __required(library, "GAPICs")
8098

8199
parsed_gapics = list()
100+
if not gapics:
101+
raise ValueError(f"GAPICs is None in {library}.")
82102
for gapic in gapics:
83-
proto_path = __required(gapic, "proto_path")
103+
proto_path = __required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER)
84104
new_gapic = GapicConfig(proto_path)
85105
parsed_gapics.append(new_gapic)
86106

@@ -114,27 +134,40 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
114134
parsed_libraries.append(new_library)
115135

116136
parsed_config = GenerationConfig(
117-
gapic_generator_version=__required(config, "gapic_generator_version"),
137+
gapic_generator_version=__required(
138+
config, "gapic_generator_version", REPO_LEVEL_PARAMETER
139+
),
118140
grpc_version=__optional(config, "grpc_version", None),
119141
protoc_version=__optional(config, "protoc_version", None),
120-
googleapis_commitish=__required(config, "googleapis_commitish"),
121-
libraries_bom_version=__required(config, "libraries_bom_version"),
122-
owlbot_cli_image=__required(config, "owlbot_cli_image"),
123-
synthtool_commitish=__required(config, "synthtool_commitish"),
124-
template_excludes=__required(config, "template_excludes"),
142+
googleapis_commitish=__required(
143+
config, "googleapis_commitish", REPO_LEVEL_PARAMETER
144+
),
145+
libraries_bom_version=__required(
146+
config, "libraries_bom_version", REPO_LEVEL_PARAMETER
147+
),
148+
owlbot_cli_image=__required(config, "owlbot_cli_image", REPO_LEVEL_PARAMETER),
149+
synthtool_commitish=__required(
150+
config, "synthtool_commitish", REPO_LEVEL_PARAMETER
151+
),
152+
template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER),
125153
libraries=parsed_libraries,
126154
)
127155

128156
return parsed_config
129157

130158

131-
def __required(config: Dict, key: str):
159+
def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER):
132160
if key not in config:
133-
raise ValueError(f"required key {key} not found in yaml")
161+
message = (
162+
f"{level}, {key}, is not found in {config} in yaml."
163+
if level != REPO_LEVEL_PARAMETER
164+
else f"{level}, {key}, is not found in yaml."
165+
)
166+
raise ValueError(message)
134167
return config[key]
135168

136169

137-
def __optional(config: Dict, key: str, default: any):
170+
def __optional(config: dict, key: str, default: any):
138171
if key not in config:
139172
return default
140173
return config[key]

‎sdk-platform-java/library_generation/test/cli/entry_point_unit_tests.py‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14+
import os
1415
import unittest
1516
from click.testing import CliRunner
16-
from library_generation.cli.entry_point import generate
17+
from library_generation.cli.entry_point import generate, validate_generation_config
18+
19+
script_dir = os.path.dirname(os.path.realpath(__file__))
20+
test_resource_dir = os.path.join(script_dir, "..", "resources", "test-config")
1721

1822

1923
class EntryPointTest(unittest.TestCase):
@@ -44,3 +48,32 @@ def test_entry_point_with_baseline_without_current_raise_file_exception(self):
4448
"current_generation_config is not specified when "
4549
"baseline_generation_config is specified.",
4650
)
51+
52+
def test_validate_generation_config_succeeds(
53+
self,
54+
):
55+
runner = CliRunner()
56+
# noinspection PyTypeChecker
57+
result = runner.invoke(
58+
validate_generation_config,
59+
[f"--generation-config-path={test_resource_dir}/generation_config.yaml"],
60+
)
61+
self.assertEqual(0, result.exit_code)
62+
63+
def test_validate_generation_config_with_duplicate_library_name_raise_file_exception(
64+
self,
65+
):
66+
runner = CliRunner()
67+
# noinspection PyTypeChecker
68+
result = runner.invoke(
69+
validate_generation_config,
70+
[
71+
f"--generation-config-path={test_resource_dir}/generation_config_with_duplicate_library_name.yaml"
72+
],
73+
)
74+
self.assertEqual(1, result.exit_code)
75+
self.assertEqual(SystemExit, result.exc_info[0])
76+
self.assertRegex(
77+
result.output,
78+
"have the same library name",
79+
)

‎sdk-platform-java/library_generation/test/model/generation_config_unit_test.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,153 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
133133
libraries=[library_1, library_2],
134134
)
135135
self.assertTrue(config.is_monorepo())
136+
137+
def test_validate_with_duplicate_library_name_raise_exception(self):
138+
self.assertRaisesRegex(
139+
ValueError,
140+
"the same library name",
141+
GenerationConfig,
142+
gapic_generator_version="",
143+
googleapis_commitish="",
144+
libraries_bom_version="",
145+
owlbot_cli_image="",
146+
synthtool_commitish="",
147+
template_excludes=[],
148+
libraries=[
149+
LibraryConfig(
150+
api_shortname="secretmanager",
151+
name_pretty="Secret API",
152+
product_documentation="",
153+
api_description="",
154+
gapic_configs=list(),
155+
),
156+
LibraryConfig(
157+
api_shortname="another-secret",
158+
name_pretty="Another Secret API",
159+
product_documentation="",
160+
api_description="",
161+
gapic_configs=list(),
162+
library_name="secretmanager",
163+
),
164+
],
165+
)
166+
167+
def test_from_yaml_without_gapic_generator_version_raise_exception(self):
168+
self.assertRaisesRegex(
169+
ValueError,
170+
"Repo level parameter, gapic_generator_version",
171+
from_yaml,
172+
f"{test_config_dir}/config_without_generator.yaml",
173+
)
174+
175+
def test_from_yaml_without_googleapis_commitish_raise_exception(self):
176+
self.assertRaisesRegex(
177+
ValueError,
178+
"Repo level parameter, googleapis_commitish",
179+
from_yaml,
180+
f"{test_config_dir}/config_without_googleapis.yaml",
181+
)
182+
183+
def test_from_yaml_without_libraries_bom_version_raise_exception(self):
184+
self.assertRaisesRegex(
185+
ValueError,
186+
"Repo level parameter, libraries_bom_version",
187+
from_yaml,
188+
f"{test_config_dir}/config_without_libraries_bom_version.yaml",
189+
)
190+
191+
def test_from_yaml_without_owlbot_cli_image_raise_exception(self):
192+
self.assertRaisesRegex(
193+
ValueError,
194+
"Repo level parameter, owlbot_cli_image",
195+
from_yaml,
196+
f"{test_config_dir}/config_without_owlbot.yaml",
197+
)
198+
199+
def test_from_yaml_without_synthtool_commitish_raise_exception(self):
200+
self.assertRaisesRegex(
201+
ValueError,
202+
"Repo level parameter, synthtool_commitish",
203+
from_yaml,
204+
f"{test_config_dir}/config_without_synthtool.yaml",
205+
)
206+
207+
def test_from_yaml_without_template_excludes_raise_exception(self):
208+
self.assertRaisesRegex(
209+
ValueError,
210+
"Repo level parameter, template_excludes",
211+
from_yaml,
212+
f"{test_config_dir}/config_without_temp_excludes.yaml",
213+
)
214+
215+
def test_from_yaml_without_libraries_raise_exception(self):
216+
self.assertRaisesRegex(
217+
ValueError,
218+
"Repo level parameter, libraries",
219+
from_yaml,
220+
f"{test_config_dir}/config_without_libraries.yaml",
221+
)
222+
223+
def test_from_yaml_without_api_shortname_raise_exception(self):
224+
self.assertRaisesRegex(
225+
ValueError,
226+
"Library level parameter, api_shortname",
227+
from_yaml,
228+
f"{test_config_dir}/config_without_api_shortname.yaml",
229+
)
230+
231+
def test_from_yaml_without_api_description_raise_exception(self):
232+
self.assertRaisesRegex(
233+
ValueError,
234+
r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*",
235+
from_yaml,
236+
f"{test_config_dir}/config_without_api_description.yaml",
237+
)
238+
239+
def test_from_yaml_without_name_pretty_raise_exception(self):
240+
self.assertRaisesRegex(
241+
ValueError,
242+
r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*",
243+
from_yaml,
244+
f"{test_config_dir}/config_without_name_pretty.yaml",
245+
)
246+
247+
def test_from_yaml_without_product_documentation_raise_exception(self):
248+
self.assertRaisesRegex(
249+
ValueError,
250+
r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*",
251+
from_yaml,
252+
f"{test_config_dir}/config_without_product_docs.yaml",
253+
)
254+
255+
def test_from_yaml_without_gapics_raise_exception(self):
256+
self.assertRaisesRegex(
257+
ValueError,
258+
"Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*",
259+
from_yaml,
260+
f"{test_config_dir}/config_without_gapics_key.yaml",
261+
)
262+
263+
def test_from_yaml_without_proto_path_raise_exception(self):
264+
self.assertRaisesRegex(
265+
ValueError,
266+
"GAPIC level parameter, proto_path",
267+
from_yaml,
268+
f"{test_config_dir}/config_without_proto_path.yaml",
269+
)
270+
271+
def test_from_yaml_with_zero_library_raise_exception(self):
272+
self.assertRaisesRegex(
273+
ValueError,
274+
"Library is None",
275+
from_yaml,
276+
f"{test_config_dir}/config_without_library_value.yaml",
277+
)
278+
279+
def test_from_yaml_with_zero_proto_path_raise_exception(self):
280+
self.assertRaisesRegex(
281+
ValueError,
282+
r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*",
283+
from_yaml,
284+
f"{test_config_dir}/config_without_gapics_value.yaml",
285+
)

‎sdk-platform-java/library_generation/test/resources/test-config/config_without_gapics_key.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
gapic_generator_version: 2.34.0
2+
libraries:
3+
- api_shortname: apigeeconnect

‎sdk-platform-java/library_generation/test/resources/test-config/config_without_gapics_value.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
gapic_generator_version: 2.34.0
2+
libraries:
3+
- api_shortname: apigeeconnect
4+
GAPICs:

sdk-platform-java/library_generation/test/resources/test-config/config_without_gapics.yaml renamed to sdk-platform-java/library_generation/test/resources/test-config/config_without_library_value.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
gapic_generator_version: 2.34.0
22
libraries:
3-
random_key:

0 commit comments

Comments
 (0)