Visitar URL original
feat(generator): delegate REST transcoding to google-api-core (#17766) · googleapis/google-cloud-python@7b05aab · GitHub
Skip to content

Commit 7b05aab

Browse files
hebaalazzehhebaalazzeh
authored
feat(generator): delegate REST transcoding to google-api-core (#17766)
this PR refactors generator templates to delegate REST transcoding to the centralized helper in `google-api-core`. Scope of Changes: - **`_shared_macros.j2`**: Replaced inline static helper invocation with `gapic_v1.rest_transcoding.transcode()`. - **`rest_base.py.j2`**: Stripped the redundant `_get_transcoded_request`, `_get_request_body_json`, and `_get_query_params_json` helper functions. - **`_rest_mixins_base.py.j2`**: Stripped the redundant static helpers from API REST mixins base. - **`rest.py.j2` & `rest_asyncio.py.j2`**: Passed the configuration flags (like `opts.rest_numeric_enums`) into the transcoding macro. - **`noxfile.py`**: Configured local `packages/google-api-core` to be installed during virtualenv test generation so the new transcoding runtime module is available. related PR: #17765
1 parent 8c26b38 commit 7b05aab

46 files changed

Lines changed: 4309 additions & 4416 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/rest_base.py.j2‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,10 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport):
124124
"{{ req_field.name | camel_case }}" : {% if req_field.field_pb.type == 9 %}"{{req_field.field_pb.default_value }}"{% elif req_field.field_pb.type in [11, 14] %}{}{% else %}{{ req_field.type.python_type(req_field.field_pb.default_value or 0) }}{% endif %},{# default is str #}
125125
{% endfor %}
126126
}
127-
128-
@classmethod
129-
def _get_unset_required_fields(cls, message_dict):
130-
return {k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict}
131127
{% endif %}{# required fields #}
132-
128+
133129
{% set method_http_options = method.http_options %}
134-
130+
135131
{{ shared_macros.http_options_method(method_http_options)|indent(8) }}
136132

137133
@staticmethod
@@ -166,7 +162,8 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport):
166162
use_integers_for_enums={{ opts.rest_numeric_enums }},
167163
))
168164
{% if method.input.required_fields %}
169-
query_params.update(_Base{{ service.name }}RestTransport._Base{{method.name}}._get_unset_required_fields(query_params))
165+
default_values = _Base{{ service.name }}RestTransport._Base{{method.name}}._Base{{method.name}}__REQUIRED_FIELDS_DEFAULT_VALUES
166+
query_params.update({k: v for k, v in default_values.items() if k not in query_params})
170167
{% endif %}{# required fields #}
171168

172169
{% if opts.rest_numeric_enums %}

‎packages/gapic-generator/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1339,7 +1339,12 @@ def test_{{ method_name }}_rest_required_fields(request_type={{ method.input.ide
13391339
assert "{{ field_name }}" not in jsonified_request
13401340
{% endfor %}
13411341

1342-
unset_fields = transport_class(credentials=ga_credentials.AnonymousCredentials()).{{ method.transport_safe_name | snake_case }}._get_unset_required_fields(jsonified_request)
1342+
default_values = getattr(
1343+
transport_class._Base{{ method.name }},
1344+
"_Base{{ method.name }}__REQUIRED_FIELDS_DEFAULT_VALUES",
1345+
{},
1346+
)
1347+
unset_fields = {k: v for k, v in default_values.items() if k not in jsonified_request}
13431348
jsonified_request.update(unset_fields)
13441349

13451350
# verify required fields with default values are now present
@@ -1355,12 +1360,10 @@ def test_{{ method_name }}_rest_required_fields(request_type={{ method.input.ide
13551360
jsonified_request["{{ field_name }}"] = {{ mock_value }}
13561361
{% endfor %}
13571362

1358-
unset_fields = transport_class(credentials=ga_credentials.AnonymousCredentials()).{{ method.transport_safe_name | snake_case }}._get_unset_required_fields(jsonified_request)
13591363
{% if method.query_params %}
13601364
# Check that path parameters and body parameters are not mixing in.
1361-
assert not set(unset_fields) - set(({% for param in method.query_params|sort %}"{{param}}", {% endfor %}))
1365+
assert not set(unset_fields) - set(({% for param in method.query_params|sort %}"{{param | camel_case}}", {% endfor %}))
13621366
{% endif %}
1363-
jsonified_request.update(unset_fields)
13641367

13651368
# verify required fields with non-default values are left alone
13661369
{% for req_field in method.input.required_fields if req_field.is_primitive %}
@@ -1460,13 +1463,6 @@ def test_{{ method_name }}_rest_required_fields(request_type={{ method.input.ide
14601463
assert expected_params == actual_params
14611464

14621465

1463-
def test_{{ method_name }}_rest_unset_required_fields():
1464-
transport = transports.{{ service.rest_transport_name }}(credentials=ga_credentials.AnonymousCredentials)
1465-
1466-
unset_fields = transport.{{ method.transport_safe_name|snake_case }}._get_unset_required_fields({})
1467-
assert set(unset_fields) == (set(({% for param in method.query_params|sort %}"{{ param|camel_case }}", {% endfor %})) & set(({% for param in method.input.required_fields %}"{{param.name|camel_case}}", {% endfor %})))
1468-
1469-
14701466
{% endif %}{# required_fields #}
14711467

14721468
{% if not method.client_streaming %}

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,24 @@ falling back to the local implementation if not present. #}
1313
{# TODO(https://github.com/googleapis/google-cloud-python/issues/17883):
1414
Backfill compatibility functions being removed from the client layer. #}
1515

16-
from typing import Optional
16+
import json
17+
{% if has_auto_populated_fields %}
18+
import uuid
19+
{% endif %}
20+
21+
from typing import Any, Dict, List, Optional, Tuple
22+
{% if has_auto_populated_fields %}
23+
from typing import Union
24+
{% endif %}
25+
26+
from google.api_core import path_template
27+
from google.protobuf import json_format
1728
from urllib.parse import urlparse, urlunparse
1829

1930
from google.auth.exceptions import MutualTLSChannelError
2031
from google.api_core.universe import EmptyUniverseError
2132

2233
{% if has_auto_populated_fields %}
23-
import uuid
24-
from typing import Union
2534
import google.protobuf.message
2635
{% endif %}
2736

@@ -189,4 +198,62 @@ def setup_request_id(
189198
setattr(request, field_name, str(uuid.uuid4()))
190199

191200
{% endif %}
201+
202+
def transcode_request(
203+
http_options: List[Dict[str, str]],
204+
request: Any,
205+
required_fields_default_values: Optional[Dict[str, Any]] = None,
206+
rest_numeric_enums: bool = False,
207+
) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]:
208+
"""Transcodes a request into HTTP method, URI, body, and query parameters.
209+
210+
Args:
211+
http_options (List[Dict[str, str]]): List of HTTP transcoding rules.
212+
request (Any): The protobuf or proto-plus request message.
213+
required_fields_default_values (Optional[Dict[str, Any]]): Dictionary
214+
of required fields default values to merge into query parameters if missing.
215+
rest_numeric_enums (bool): Whether to encode enums as integers.
216+
217+
Returns:
218+
Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: A tuple containing:
219+
- The raw transcoded request dictionary (containing keys like 'uri', 'method').
220+
- The serialized request body JSON string, or None if no body.
221+
- The query parameters dictionary.
222+
"""
223+
if request is None:
224+
raise TypeError("request cannot be None")
225+
226+
# Convert proto-plus message to its underlying protobuf message if needed
227+
pb_request = getattr(request, "_pb", request)
228+
229+
transcoded_request = path_template.transcode(http_options, pb_request)
230+
231+
body_json = None
232+
if transcoded_request.get("body") is not None:
233+
body_json = json_format.MessageToJson(
234+
transcoded_request["body"],
235+
use_integers_for_enums=rest_numeric_enums,
236+
)
237+
238+
query_params_json = {}
239+
if transcoded_request.get("query_params") is not None:
240+
query_params_json = json.loads(
241+
json_format.MessageToJson(
242+
transcoded_request["query_params"],
243+
use_integers_for_enums=rest_numeric_enums,
244+
)
245+
)
246+
247+
# If required_fields_default_values is provided, we merge default values for missing
248+
# required fields into the query parameters.
249+
if required_fields_default_values:
250+
for k, v in required_fields_default_values.items():
251+
if k not in query_params_json:
252+
query_params_json[k] = v
253+
254+
if rest_numeric_enums:
255+
query_params_json["$alt"] = "json;enum-encoding=int"
256+
257+
return transcoded_request, body_json, query_params_json
258+
192259
{% endblock %}

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -194,22 +194,24 @@ def _get_http_options():
194194
service: The service.
195195
is_async (bool): Used to determine the code path i.e. whether for sync or async call.
196196
is_request_message_proto_plus_type (bool): Used to determine whether the request message is a proto-plus type. #}
197-
{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False) %}
197+
{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False, rest_numeric_enums=False) %}
198198
{% set service_name = service.name %}
199199
{% set await_prefix = "await " if is_async else "" %}
200200
{% set async_class_prefix = "Async" if is_async else "" %}
201201

202202
http_options = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_http_options()
203-
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2274): Add debug log before intercepting a request #}
203+
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2274): Add debug log before intercepting a request #}
204204
request, metadata = {{ await_prefix }}self._interceptor.pre_{{ method_name|snake_case }}(request, metadata)
205-
transcoded_request = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_transcoded_request(http_options, request)
206-
207-
{% if body_spec %}
208-
body = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_request_body_json(transcoded_request)
209-
{% endif %}{# body_spec #}
210-
211-
# Jsonify the query params
212-
query_params = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_query_params_json(transcoded_request)
205+
transcoded_request, body, query_params = transcode_request(
206+
http_options,
207+
request,
208+
required_fields_default_values=getattr(
209+
_Base{{ service_name }}RestTransport._Base{{method_name}},
210+
"_Base{{method_name}}__REQUIRED_FIELDS_DEFAULT_VALUES",
211+
None,
212+
),
213+
rest_numeric_enums={{ rest_numeric_enums }},
214+
)
213215

214216
if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(logging.DEBUG): # pragma: NO COVER
215217
request_url = "{host}{uri}".format(host=self._host, uri=transcoded_request['uri'])

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,5 @@
2525

2626
{{ shared_macros.http_options_method(api.mixin_http_options["{}".format(name)])|indent(8)}}
2727

28-
@staticmethod
29-
def _get_transcoded_request(http_options, request):
30-
request_kwargs = json_format.MessageToDict(request)
31-
transcoded_request = path_template.transcode(
32-
http_options, **request_kwargs)
33-
return transcoded_request
34-
35-
{% set body_spec = api.mixin_http_options["{}".format(name)][0].body %}
36-
{%- if body_spec %}
37-
38-
@staticmethod
39-
def _get_request_body_json(transcoded_request):
40-
body = json.dumps(transcoded_request['body'])
41-
return body
42-
43-
{%- endif %} {# body_spec #}
44-
45-
@staticmethod
46-
def _get_query_params_json(transcoded_request):
47-
query_params = json.loads(json.dumps(transcoded_request['query_params']))
48-
return query_params
49-
5028
{% endfor %}
5129
{% endif %} {# rest in opts.transport #}

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ from google.api_core import retry as retries
1313
from google.api_core import rest_helpers
1414
from google.api_core import rest_streaming
1515
from google.api_core import gapic_v1
16+
{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}
17+
from {{package_path}}._compat import transcode_request
1618
import google.protobuf
1719

1820
from google.protobuf import json_format
@@ -245,7 +247,7 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport):
245247
{% endif %}
246248
"""
247249

248-
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type)|indent(8) }}
250+
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }}
249251

250252
{% if not method.void %}
251253
# Return the response

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ from google.cloud.location import locations_pb2 # type: ignore
3434
from google.api_core import retry_async as retries
3535
from google.api_core import rest_helpers
3636
from google.api_core import rest_streaming_async # type: ignore
37+
{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}
38+
from {{package_path}}._compat import transcode_request
39+
3740
import google.protobuf
3841

3942
from google.protobuf import json_format
@@ -203,7 +206,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport):
203206
{% endif %}
204207
"""
205208

206-
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type)|indent(8) }}
209+
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }}
207210

208211
{% if not method.void %}
209212
# Return the response

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2‎

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -116,55 +116,11 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport):
116116
{% endfor %}
117117
}
118118

119-
@classmethod
120-
def _get_unset_required_fields(cls, message_dict):
121-
return {k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict}
122119
{% endif %}{# required fields #}
123-
120+
124121
{% set method_http_options = method.http_options %}
125-
126-
{{ shared_macros.http_options_method(method_http_options)|indent(8) }}
127-
128-
@staticmethod
129-
def _get_transcoded_request(http_options, request):
130-
{% if method.input.ident.is_proto_plus_type %}
131-
pb_request = {{method.input.ident}}.pb(request)
132-
{% else %}
133-
pb_request = request
134-
{% endif %}
135-
transcoded_request = path_template.transcode(http_options, pb_request)
136-
return transcoded_request
137-
138-
{% set body_spec = method.http_options[0].body %}
139-
{%- if body_spec %}
140-
141-
@staticmethod
142-
def _get_request_body_json(transcoded_request):
143-
# Jsonify the request body
144-
145-
body = json_format.MessageToJson(
146-
transcoded_request['body'],
147-
use_integers_for_enums={{ opts.rest_numeric_enums }}
148-
)
149-
return body
150-
151-
{%- endif %}{# body_spec #}
152-
153-
@staticmethod
154-
def _get_query_params_json(transcoded_request):
155-
query_params = json.loads(json_format.MessageToJson(
156-
transcoded_request['query_params'],
157-
use_integers_for_enums={{ opts.rest_numeric_enums }},
158-
))
159-
{% if method.input.required_fields %}
160-
query_params.update(_Base{{ service.name }}RestTransport._Base{{method.name}}._get_unset_required_fields(query_params))
161-
{% endif %}{# required fields #}
162-
163-
{% if opts.rest_numeric_enums %}
164-
query_params["$alt"] = "json;enum-encoding=int"
165-
{% endif %}
166-
return query_params
167122

123+
{{ shared_macros.http_options_method(method_http_options)|indent(8) }}
168124
{% endif %}{# method.http_options and not method.client_streaming #}
169125
{% endfor %}
170126

0 commit comments

Comments
 (0)