Visitar URL original
fix: Use join keys instead of entity names in ODFV materialization (#… · feast-dev/feast@abffebc · GitHub
Skip to content

Commit abffebc

Browse files
anshishrivastavaanshishrivastava
authored
fix: Use join keys instead of entity names in ODFV materialization (#6645)
* fix: Use join keys instead of entity names in ODFV materialization _materialize_odfv used FeatureView.entities (entity names) instead of FeatureView.join_keys (actual join key column names) when building the entity DataFrame and querying the offline store during ODFV materialization. When an Entity's name differs from its join_keys (a supported, documented pattern), this caused materialization to query for a nonexistent column, breaking materialize()/materialize_incremental() for any OnDemandFeatureView with write_to_online_store=True sourced from such a feature view. Fixes #5965 Signed-off-by: Anshi Shrivastava <anshi4shrivastava@gmail.com> * style: Apply ruff format Signed-off-by: Anshi Shrivastava <anshi4shrivastava@gmail.com> --------- Signed-off-by: Anshi Shrivastava <anshi4shrivastava@gmail.com>
1 parent 40fb788 commit abffebc

2 files changed

Lines changed: 68 additions & 3 deletions

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2341,7 +2341,7 @@ def _materialize_odfv(
23412341
}
23422342

23432343
for source_fv in source_fvs:
2344-
all_join_keys.update(source_fv.entities)
2344+
all_join_keys.update(source_fv.join_keys)
23452345
if source_fv.batch_source:
23462346
entity_timestamp_col_names.add(source_fv.batch_source.timestamp_field)
23472347

@@ -2381,7 +2381,7 @@ def _materialize_odfv(
23812381
job = provider.offline_store.pull_latest_from_table_or_query(
23822382
config=self.config,
23832383
data_source=source_fv.batch_source,
2384-
join_key_columns=source_fv.entities,
2384+
join_key_columns=source_fv.join_keys,
23852385
feature_name_columns=[f.name for f in source_fv.features],
23862386
timestamp_field=source_fv.batch_source.timestamp_field,
23872387
created_timestamp_column=getattr(

‎sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from datetime import datetime, timedelta
1+
from datetime import datetime, timedelta, timezone
22
from tempfile import mkstemp
33
from unittest.mock import AsyncMock, Mock, patch
44

@@ -19,11 +19,13 @@
1919
from feast.infra.offline_stores.file_source import FileSource
2020
from feast.infra.online_stores.dynamodb import DynamoDBOnlineStore
2121
from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
22+
from feast.on_demand_feature_view import OnDemandFeatureView
2223
from feast.permissions.action import AuthzedAction
2324
from feast.permissions.permission import Permission
2425
from feast.permissions.policy import RoleBasedPolicy
2526
from feast.repo_config import RegistryConfig, RepoConfig
2627
from feast.stream_feature_view import stream_feature_view
28+
from feast.transformation.pandas_transformation import PandasTransformation
2729
from feast.types import Array, Bytes, Float32, Int64, String, ValueType, from_value_type
2830
from tests.universal.feature_repos.universal.feature_views import TAGS
2931
from tests.utils.cli_repo_creator import CliRunner, get_example_repo
@@ -511,6 +513,69 @@ def test_reapply_feature_view(test_feature_store, dataframe_source):
511513
test_feature_store.teardown()
512514

513515

516+
@pytest.mark.parametrize(
517+
"test_feature_store",
518+
[lazy_fixture("feature_store_with_local_registry")],
519+
)
520+
@pytest.mark.parametrize("dataframe_source", [lazy_fixture("simple_dataset_1")])
521+
def test_materialize_incremental_odfv_entity_name_differs_from_join_key(
522+
test_feature_store, dataframe_source
523+
):
524+
"""Regression test for https://github.com/feast-dev/feast/issues/5965.
525+
526+
_materialize_odfv must resolve source feature views' join key *columns*
527+
(FeatureView.join_keys), not their entity *names* (FeatureView.entities),
528+
when building the entity_df and querying the offline store. Before the
529+
fix, this failed whenever an Entity's name differed from its join_keys.
530+
"""
531+
with prep_file_source(df=dataframe_source, timestamp_field="ts_1") as file_source:
532+
# Entity name ("id") intentionally differs from its join key
533+
# ("id_join_key"), mirroring the exact repro from the issue.
534+
e = Entity(name="id", join_keys=["id_join_key"])
535+
536+
source_fv = FeatureView(
537+
name="my_feature_view_1",
538+
schema=[Field(name="float_col", dtype=Float32)],
539+
entities=[e],
540+
source=file_source,
541+
ttl=timedelta(days=3650),
542+
)
543+
544+
def transform(features_df: pd.DataFrame) -> pd.DataFrame:
545+
out = pd.DataFrame()
546+
out["label"] = features_df["float_col"].apply(
547+
lambda v: "high" if v >= 1 else "low"
548+
)
549+
return out
550+
551+
odfv = OnDemandFeatureView(
552+
name="my_odfv",
553+
entities=[e],
554+
sources=[source_fv],
555+
schema=[Field(name="label", dtype=String)],
556+
feature_transformation=PandasTransformation(
557+
udf=transform, udf_string="transform"
558+
),
559+
write_to_online_store=True,
560+
)
561+
562+
test_feature_store.apply([e, source_fv, odfv])
563+
564+
# Should not raise. Before the fix, this raised
565+
# FeastJoinKeysDuringMaterialization because _materialize_odfv
566+
# queried the offline store for a column named "id" (the entity
567+
# name) instead of "id_join_key" (the actual join key column).
568+
test_feature_store.materialize_incremental(end_date=datetime.now(timezone.utc))
569+
570+
response = test_feature_store.get_online_features(
571+
features=["my_odfv:label"],
572+
entity_rows=[{"id_join_key": 1}],
573+
).to_dict()
574+
assert response["label"][0] is not None
575+
576+
test_feature_store.teardown()
577+
578+
514579
def test_apply_conflicting_feature_view_names(feature_store_with_local_registry):
515580
"""Test applying feature views with non-case-insensitively unique names"""
516581
driver = Entity(name="driver", join_keys=["driver_id"])

0 commit comments

Comments
 (0)