Visitar URL original
feat: Updating protos to separate transformation (#4018) · feast-dev/feast@c58ef74 · GitHub
Skip to content

Commit c58ef74

Browse files
franciscojavierarceofranciscojavierarceo
authored
feat: Updating protos to separate transformation (#4018)
* feat: updating protos to separate transformation Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * fixed stuff...i think Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated tests and registry diff function Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated base registry Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated react component Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * formatted Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated stream feature view proto Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * making the proto changes backwards compatable Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * trying to make this backwards compatible Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * caught a bug and fixed the linter * actually linted Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated ui component Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * accidentally commented out fixtures * Updated Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * incrementing protos * updated tests Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * fixed linting issue and made backwards compatible Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * added more tests Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> --------- Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com>
1 parent e815562 commit c58ef74

14 files changed

Lines changed: 482 additions & 232 deletions

‎protos/feast/core/OnDemandFeatureView.proto‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import "feast/core/FeatureView.proto";
2727
import "feast/core/FeatureViewProjection.proto";
2828
import "feast/core/Feature.proto";
2929
import "feast/core/DataSource.proto";
30+
import "feast/core/Transformation.proto";
3031

3132
message OnDemandFeatureView {
3233
// User-specified specifications of this feature view.
@@ -49,9 +50,11 @@ message OnDemandFeatureViewSpec {
4950
map<string, OnDemandSource> sources = 4;
5051

5152
oneof transformation {
52-
UserDefinedFunction user_defined_function = 5;
53-
OnDemandSubstraitTransformation on_demand_substrait_transformation = 9;
53+
UserDefinedFunction user_defined_function = 5 [deprecated = true];
54+
OnDemandSubstraitTransformation on_demand_substrait_transformation = 9 [deprecated = true];
5455
}
56+
// Oneof with {user_defined_function, on_demand_substrait_transformation}
57+
FeatureTransformationV2 feature_transformation = 10;
5558

5659
// Description of the on demand feature view.
5760
string description = 6;
@@ -61,6 +64,7 @@ message OnDemandFeatureViewSpec {
6164

6265
// Owner of the on demand feature view.
6366
string owner = 8;
67+
string mode = 11;
6468
}
6569

6670
message OnDemandFeatureViewMeta {
@@ -81,6 +85,8 @@ message OnDemandSource {
8185

8286
// Serialized representation of python function.
8387
message UserDefinedFunction {
88+
option deprecated = true;
89+
8490
// The function name
8591
string name = 1;
8692

@@ -92,5 +98,7 @@ message UserDefinedFunction {
9298
}
9399

94100
message OnDemandSubstraitTransformation {
101+
option deprecated = true;
102+
95103
bytes substrait_plan = 1;
96-
}
104+
}

‎protos/feast/core/StreamFeatureView.proto‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import "feast/core/FeatureView.proto";
2929
import "feast/core/Feature.proto";
3030
import "feast/core/DataSource.proto";
3131
import "feast/core/Aggregation.proto";
32+
import "feast/core/Transformation.proto";
3233

3334
message StreamFeatureView {
3435
// User-specified specifications of this feature view.
@@ -77,7 +78,8 @@ message StreamFeatureViewSpec {
7778
bool online = 12;
7879

7980
// Serialized function that is encoded in the streamfeatureview
80-
UserDefinedFunction user_defined_function = 13;
81+
UserDefinedFunction user_defined_function = 13 [deprecated = true];
82+
8183

8284
// Mode of execution
8385
string mode = 14;
@@ -87,5 +89,8 @@ message StreamFeatureViewSpec {
8789

8890
// Timestamp field for aggregation
8991
string timestamp_field = 16;
92+
93+
// Oneof with {user_defined_function, on_demand_substrait_transformation}
94+
FeatureTransformationV2 feature_transformation = 17;
9095
}
9196

‎protos/feast/core/Transformation.proto‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
syntax = "proto3";
2+
package feast.core;
3+
4+
option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
5+
option java_outer_classname = "FeatureTransformationProto";
6+
option java_package = "feast.proto.core";
7+
8+
import "google/protobuf/duration.proto";
9+
10+
// Serialized representation of python function.
11+
message UserDefinedFunctionV2 {
12+
// The function name
13+
string name = 1;
14+
15+
// The python-syntax function body (serialized by dill)
16+
bytes body = 2;
17+
18+
// The string representation of the udf
19+
string body_text = 3;
20+
}
21+
22+
// A feature transformation executed as a user-defined function
23+
message FeatureTransformationV2 {
24+
// Note this Transformation starts at 5 for backwards compatibility
25+
oneof transformation {
26+
UserDefinedFunctionV2 user_defined_function = 1;
27+
OnDemandSubstraitTransformationV2 on_demand_substrait_transformation = 2;
28+
}
29+
}
30+
31+
message OnDemandSubstraitTransformationV2 {
32+
bytes substrait_plan = 1;
33+
}

‎sdk/python/feast/diff/registry_diff.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import warnings
12
from dataclasses import dataclass
23
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, cast
34

@@ -144,11 +145,25 @@ def diff_registry_objects(
144145
if _field.name in FIELDS_TO_IGNORE:
145146
continue
146147
elif getattr(current_spec, _field.name) != getattr(new_spec, _field.name):
147-
if _field.name == "user_defined_function":
148+
# TODO: Delete "transformation" after we've safely deprecated it from the proto
149+
if _field.name in ["transformation", "feature_transformation"]:
150+
warnings.warn(
151+
"transformation will be deprecated in the future please use feature_transformation instead.",
152+
DeprecationWarning,
153+
)
148154
current_spec = cast(OnDemandFeatureViewSpec, current_spec)
149155
new_spec = cast(OnDemandFeatureViewSpec, new_spec)
150-
current_udf = current_spec.user_defined_function
151-
new_udf = new_spec.user_defined_function
156+
# Check if the old proto is populated and use that if it is
157+
deprecated_udf = current_spec.user_defined_function
158+
feature_transformation_udf = (
159+
current_spec.feature_transformation.user_defined_function
160+
)
161+
current_udf = (
162+
deprecated_udf
163+
if deprecated_udf.body_text != ""
164+
else feature_transformation_udf
165+
)
166+
new_udf = new_spec.feature_transformation.user_defined_function
152167
for _udf_field in current_udf.DESCRIPTOR.fields:
153168
if _udf_field.name == "body":
154169
continue

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
import json
15+
import warnings
1516
from abc import ABC, abstractmethod
1617
from collections import defaultdict
1718
from datetime import datetime
@@ -662,10 +663,16 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]:
662663
key=lambda on_demand_feature_view: on_demand_feature_view.name,
663664
):
664665
odfv_dict = self._message_to_sorted_dict(on_demand_feature_view.to_proto())
665-
666-
odfv_dict["spec"]["userDefinedFunction"][
666+
# We are logging a warning because the registry object may be read from a proto that is not updated
667+
# i.e., we have to submit dual writes but in order to ensure the read behavior succeeds we have to load
668+
# both objects to compare any changes in the registry
669+
warnings.warn(
670+
"We will be deprecating the usage of spec.userDefinedFunction in a future release please upgrade cautiously.",
671+
DeprecationWarning,
672+
)
673+
odfv_dict["spec"]["featureTransformation"]["userDefinedFunction"][
667674
"body"
668-
] = on_demand_feature_view.transformation.udf_string
675+
] = on_demand_feature_view.feature_transformation.udf_string
669676
registry_dict["onDemandFeatureViews"].append(odfv_dict)
670677
for request_feature_view in sorted(
671678
self.list_request_feature_views(project=project),
@@ -684,6 +691,7 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]:
684691
"body"
685692
] = stream_feature_view.udf_string
686693
registry_dict["streamFeatureViews"].append(sfv_dict)
694+
687695
for saved_dataset in sorted(
688696
self.list_saved_datasets(project=project), key=lambda item: item.name
689697
):

‎sdk/python/feast/on_demand_feature_view.py‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@
2727
OnDemandFeatureViewSpec,
2828
OnDemandSource,
2929
)
30+
from feast.protos.feast.core.Transformation_pb2 import (
31+
FeatureTransformationV2 as FeatureTransformationProto,
32+
)
33+
from feast.protos.feast.core.Transformation_pb2 import (
34+
UserDefinedFunctionV2 as UserDefinedFunctionProto,
35+
)
3036
from feast.type_map import (
3137
feast_value_type_to_pandas_type,
3238
python_type_to_feast_value_type,
@@ -63,6 +69,7 @@ class OnDemandFeatureView(BaseFeatureView):
6369
source_feature_view_projections: Dict[str, FeatureViewProjection]
6470
source_request_sources: Dict[str, RequestSource]
6571
transformation: Union[OnDemandPandasTransformation]
72+
feature_transformation: Union[OnDemandPandasTransformation]
6673
description: str
6774
tags: Dict[str, str]
6875
owner: str
@@ -83,6 +90,7 @@ def __init__( # noqa: C901
8390
udf: Optional[FunctionType] = None,
8491
udf_string: str = "",
8592
transformation: Optional[Union[OnDemandPandasTransformation]] = None,
93+
feature_transformation: Optional[Union[OnDemandPandasTransformation]] = None,
8694
description: str = "",
8795
tags: Optional[Dict[str, str]] = None,
8896
owner: str = "",
@@ -101,6 +109,7 @@ def __init__( # noqa: C901
101109
dataframes as inputs.
102110
udf_string (deprecated): The source code version of the udf (for diffing and displaying in Web UI)
103111
transformation: The user defined transformation.
112+
feature_transformation: The user defined transformation.
104113
description (optional): A human-readable description.
105114
tags (optional): A dictionary of key-value pairs to store arbitrary metadata.
106115
owner (optional): The owner of the on demand feature view, typically the email
@@ -139,6 +148,7 @@ def __init__( # noqa: C901
139148
] = odfv_source.projection
140149

141150
self.transformation = transformation
151+
self.feature_transformation = self.transformation
142152

143153
@property
144154
def proto_class(self) -> Type[OnDemandFeatureViewProto]:
@@ -151,6 +161,7 @@ def __copy__(self):
151161
sources=list(self.source_feature_view_projections.values())
152162
+ list(self.source_request_sources.values()),
153163
transformation=self.transformation,
164+
feature_transformation=self.transformation,
154165
description=self.description,
155166
tags=self.tags,
156167
owner=self.owner,
@@ -172,6 +183,7 @@ def __eq__(self, other):
172183
!= other.source_feature_view_projections
173184
or self.source_request_sources != other.source_request_sources
174185
or self.transformation != other.transformation
186+
or self.feature_transformation != other.feature_transformation
175187
):
176188
return False
177189

@@ -205,16 +217,19 @@ def to_proto(self) -> OnDemandFeatureViewProto:
205217
request_data_source=request_sources.to_proto()
206218
)
207219

208-
spec = OnDemandFeatureViewSpec(
209-
name=self.name,
210-
features=[feature.to_proto() for feature in self.features],
211-
sources=sources,
220+
feature_transformation = FeatureTransformationProto(
212221
user_defined_function=self.transformation.to_proto()
213222
if type(self.transformation) == OnDemandPandasTransformation
214223
else None,
215-
on_demand_substrait_transformation=self.transformation.to_proto() # type: ignore
224+
on_demand_substrait_transformation=self.transformation.to_proto()
216225
if type(self.transformation) == OnDemandSubstraitTransformation
217-
else None,
226+
else None, # type: ignore
227+
)
228+
spec = OnDemandFeatureViewSpec(
229+
name=self.name,
230+
features=[feature.to_proto() for feature in self.features],
231+
sources=sources,
232+
feature_transformation=feature_transformation,
218233
description=self.description,
219234
tags=self.tags,
220235
owner=self.owner,
@@ -254,18 +269,37 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto):
254269
)
255270

256271
if (
257-
on_demand_feature_view_proto.spec.WhichOneof("transformation")
272+
on_demand_feature_view_proto.spec.feature_transformation.WhichOneof(
273+
"transformation"
274+
)
258275
== "user_defined_function"
276+
and on_demand_feature_view_proto.spec.feature_transformation.user_defined_function.body_text
277+
!= ""
259278
):
260279
transformation = OnDemandPandasTransformation.from_proto(
261-
on_demand_feature_view_proto.spec.user_defined_function
280+
on_demand_feature_view_proto.spec.feature_transformation.user_defined_function
262281
)
263282
elif (
264-
on_demand_feature_view_proto.spec.WhichOneof("transformation")
283+
on_demand_feature_view_proto.spec.feature_transformation.WhichOneof(
284+
"transformation"
285+
)
265286
== "on_demand_substrait_transformation"
266287
):
267288
transformation = OnDemandSubstraitTransformation.from_proto(
268-
on_demand_feature_view_proto.spec.on_demand_substrait_transformation
289+
on_demand_feature_view_proto.spec.feature_transformation.on_demand_substrait_transformation
290+
)
291+
elif (
292+
hasattr(on_demand_feature_view_proto.spec, "user_defined_function")
293+
and on_demand_feature_view_proto.spec.feature_transformation.user_defined_function.body_text
294+
== ""
295+
):
296+
backwards_compatible_udf = UserDefinedFunctionProto(
297+
name=on_demand_feature_view_proto.spec.user_defined_function.name,
298+
body=on_demand_feature_view_proto.spec.user_defined_function.body,
299+
body_text=on_demand_feature_view_proto.spec.user_defined_function.body_text,
300+
)
301+
transformation = OnDemandPandasTransformation.from_proto(
302+
user_defined_function_proto=backwards_compatible_udf,
269303
)
270304
else:
271305
raise Exception("At least one transformation type needs to be provided")

‎sdk/python/feast/on_demand_pandas_transformation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import dill
44
import pandas as pd
55

6-
from feast.protos.feast.core.OnDemandFeatureView_pb2 import (
7-
UserDefinedFunction as UserDefinedFunctionProto,
6+
from feast.protos.feast.core.Transformation_pb2 import (
7+
UserDefinedFunctionV2 as UserDefinedFunctionProto,
88
)
99

1010

‎sdk/python/feast/on_demand_substrait_transformation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
import pyarrow
33
import pyarrow.substrait as substrait # type: ignore # noqa
44

5-
from feast.protos.feast.core.OnDemandFeatureView_pb2 import (
6-
OnDemandSubstraitTransformation as OnDemandSubstraitTransformationProto,
5+
from feast.protos.feast.core.Transformation_pb2 import (
6+
OnDemandSubstraitTransformationV2 as OnDemandSubstraitTransformationProto,
77
)
88

99

0 commit comments

Comments
 (0)