Visitar URL original
feat: Retrieve historical features from BigQuery without entity_df (#… · feast-dev/feast@cd5f6bb · GitHub
Skip to content

Commit cd5f6bb

Browse files
singhhimanshu0811Himanshu Singhnquinn408ugiordanclaude
authored
feat: Retrieve historical features from BigQuery without entity_df (#6569)
* retrieval from bigquery without entity df Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * feat: Implement RegistryServer.Proto RPC with RBAC-filtered response (#6558) (#6552) Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * feat(operator): integrate cluster TLS profile for OCP 5.0 compliance Read the cluster TLS profile from apiservers.config.openshift.io/cluster at startup. Apply MinVersion, CipherSuites, and NextProtos to webhook and metrics server TLS configs. Fail closed on unexpected errors. Use Intermediate defaults on non-OpenShift clusters. This ensures the feast-operator honors the cluster-wide TLS security profile, which is required for OCP 5.0. The implementation gracefully falls back to hardened defaults (TLS 1.2, ECDHE ciphers) on non-OpenShift clusters, so it does not break vanilla Kubernetes deployments. Signed-off-by: Ugo Giordano <ugiordan@redhat.com> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Ugo Giordano <ugiordan@redhat.com> Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * fixing linter error Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * fixing unit test error Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * linter errors Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * applied recos Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * unit tests and linter error Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * pre commit checks Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * pre commit checks Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * fixing errors Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * casting null as string Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> * fixing test cases Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> --------- Signed-off-by: Himanshu Singh <himanshu.singh@walmart.com> Signed-off-by: Ugo Giordano <ugiordan@redhat.com> Co-authored-by: Himanshu Singh <himanshu.singh@walmart.com> Co-authored-by: nquinn408 <57655411+nquinn408@users.noreply.github.com> Co-authored-by: Ugo Giordano <ugiordan@redhat.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 51e3a16 commit cd5f6bb

3 files changed

Lines changed: 898 additions & 32 deletions

File tree

‎sdk/python/feast/infra/offline_stores/bigquery.py‎

Lines changed: 131 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
ContextManager,
1111
Dict,
1212
Iterator,
13+
KeysView,
1314
List,
1415
Literal,
1516
Optional,
1617
Tuple,
1718
Union,
19+
cast,
1820
)
1921

2022
import numpy as np
@@ -57,7 +59,7 @@
5759
from feast.on_demand_feature_view import OnDemandFeatureView
5860
from feast.repo_config import FeastConfigBaseModel, RepoConfig
5961
from feast.saved_dataset import SavedDatasetStorage
60-
from feast.utils import _utc_now, get_user_agent
62+
from feast.utils import _utc_now, compute_non_entity_date_range, get_user_agent
6163

6264
from .bigquery_source import (
6365
BigQueryLoggingDestination,
@@ -267,10 +269,11 @@ def get_historical_features(
267269
config: RepoConfig,
268270
feature_views: List[FeatureView],
269271
feature_refs: List[str],
270-
entity_df: Union[pd.DataFrame, str],
272+
entity_df: Optional[Union[pd.DataFrame, str]],
271273
registry: BaseRegistry,
272274
project: str,
273275
full_feature_names: bool = False,
276+
**kwargs: Any,
274277
) -> RetrievalJob:
275278
# TODO: Add entity_df validation in order to fail before interacting with BigQuery
276279
assert isinstance(config.offline_store, BigQueryOfflineStoreConfig)
@@ -297,60 +300,100 @@ def get_historical_features(
297300
config.offline_store.table_create_disposition,
298301
)
299302

300-
entity_schema = _get_entity_schema(
301-
client=client,
302-
entity_df=entity_df,
303-
)
303+
# Non-entity mode: create a left temporary table from entity keys - any entity key having an event in the time window
304304

305-
entity_df_event_timestamp_col = (
306-
offline_utils.infer_event_timestamp_from_entity_df(entity_schema)
307-
)
305+
non_entity_mode = entity_df is None
308306

309-
entity_df_event_timestamp_range = _get_entity_df_event_timestamp_range(
310-
entity_df,
311-
entity_df_event_timestamp_col,
312-
client,
313-
)
307+
if non_entity_mode:
308+
start_date, end_date = compute_non_entity_date_range(
309+
feature_views,
310+
start_date=kwargs.get("start_date"),
311+
end_date=kwargs.get("end_date"),
312+
)
313+
entity_df_event_timestamp_range = (start_date, end_date)
314314

315-
@contextlib.contextmanager
316-
def query_generator() -> Iterator[str]:
317-
_upload_entity_df(
315+
# Pre-compute query contexts to collect entity column names per feature view.
316+
fv_query_contexts_pre = offline_utils.get_feature_view_query_context(
317+
feature_refs,
318+
feature_views,
319+
registry,
320+
project,
321+
entity_df_event_timestamp_range,
322+
)
323+
all_entities = offline_utils.gather_all_entities(fv_query_contexts_pre)
324+
event_timestamp_col = "entity_ts"
325+
entity_schema_keys: KeysView[str] = cast(
326+
KeysView[str],
327+
{k: None for k in (all_entities + [event_timestamp_col])}.keys(),
328+
)
329+
330+
entity_schema = None
331+
else:
332+
entity_schema = _get_entity_schema(
318333
client=client,
319-
table_name=table_reference,
320334
entity_df=entity_df,
321335
)
322-
323-
expected_join_keys = offline_utils.get_expected_join_keys(
324-
project, feature_views, registry
336+
event_timestamp_col = offline_utils.infer_event_timestamp_from_entity_df(
337+
entity_schema
325338
)
326-
327-
offline_utils.assert_expected_columns_in_entity_df(
328-
entity_schema, expected_join_keys, entity_df_event_timestamp_col
339+
entity_df_event_timestamp_range = _get_entity_df_event_timestamp_range(
340+
entity_df,
341+
event_timestamp_col,
342+
client,
329343
)
330-
331-
# Build a query context containing all information required to template the BigQuery SQL query
332-
query_context = offline_utils.get_feature_view_query_context(
344+
entity_schema_keys = entity_schema.keys()
345+
all_entities = []
346+
start_date = entity_df_event_timestamp_range[0]
347+
end_date = entity_df_event_timestamp_range[1]
348+
fv_query_contexts_pre = offline_utils.get_feature_view_query_context(
333349
feature_refs,
334350
feature_views,
335351
registry,
336352
project,
337353
entity_df_event_timestamp_range,
338354
)
339355

356+
@contextlib.contextmanager
357+
def query_generator() -> Iterator[str]:
358+
if non_entity_mode:
359+
_bq_create_entity_union_table(
360+
client=client,
361+
table_name=table_reference,
362+
feature_views=feature_views,
363+
fv_query_contexts=fv_query_contexts_pre,
364+
start_date=start_date,
365+
end_date=end_date,
366+
all_entities=all_entities,
367+
event_timestamp_col=event_timestamp_col,
368+
)
369+
else:
370+
_upload_entity_df(
371+
client=client,
372+
table_name=table_reference,
373+
entity_df=entity_df,
374+
)
375+
expected_join_keys = offline_utils.get_expected_join_keys(
376+
project, feature_views, registry
377+
)
378+
assert entity_schema is not None
379+
offline_utils.assert_expected_columns_in_entity_df(
380+
entity_schema, expected_join_keys, event_timestamp_col
381+
)
382+
340383
# Generate the BigQuery SQL query from the query context
341384
query = offline_utils.build_point_in_time_query(
342-
query_context,
385+
feature_view_query_contexts=fv_query_contexts_pre,
343386
left_table_query_string=table_reference,
344-
entity_df_event_timestamp_col=entity_df_event_timestamp_col,
345-
entity_df_columns=entity_schema.keys(),
387+
entity_df_event_timestamp_col=event_timestamp_col,
388+
entity_df_columns=entity_schema_keys,
346389
query_template=MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN,
347390
full_feature_names=full_feature_names,
348391
)
349392

350393
try:
351394
yield query
352395
finally:
353-
# Asynchronously clean up the uploaded Bigquery table, which will expire
396+
# Asynchronously clean up the uploaded BigQuery table, which will expire
354397
# if cleanup fails
355398
client.delete_table(table=table_reference, not_found_ok=True)
356399

@@ -364,7 +407,7 @@ def query_generator() -> Iterator[str]:
364407
),
365408
metadata=RetrievalMetadata(
366409
features=feature_refs,
367-
keys=list(entity_schema.keys() - {entity_df_event_timestamp_col}),
410+
keys=list(set(entity_schema_keys) - {event_timestamp_col}),
368411
min_event_timestamp=entity_df_event_timestamp_range[0],
369412
max_event_timestamp=entity_df_event_timestamp_range[1],
370413
),
@@ -567,6 +610,62 @@ def clear_monitoring_baseline(
567610
)
568611

569612

613+
def _bq_create_entity_union_table(
614+
client: "Client",
615+
table_name: str,
616+
feature_views: List[FeatureView],
617+
fv_query_contexts: List[offline_utils.FeatureViewQueryContext],
618+
start_date: datetime,
619+
end_date: datetime,
620+
all_entities: List[str],
621+
event_timestamp_col: str,
622+
) -> None:
623+
"""
624+
Creates a BigQuery temp table containing the UNION DISTINCT of entity keys observed
625+
across all feature views in [start_date, end_date], plus a stable as-of timestamp
626+
column set to end_date. Used as the left table for PIT joins in non-entity mode.
627+
"""
628+
start_str = start_date.strftime("%Y-%m-%dT%H:%M:%S")
629+
end_str = end_date.strftime("%Y-%m-%dT%H:%M:%S")
630+
631+
per_view_selects: List[str] = []
632+
for fv, ctx in zip(feature_views, fv_query_contexts):
633+
assert isinstance(fv.batch_source, BigQuerySource)
634+
from_expression = fv.batch_source.get_table_query_string()
635+
timestamp_field = ctx.timestamp_field
636+
637+
ctx_entities_set = set(ctx.entities)
638+
select_entities: List[str] = []
639+
for col in all_entities:
640+
if col in ctx_entities_set:
641+
select_entities.append(f"`{col}`")
642+
else:
643+
select_entities.append(f"CAST(NULL AS STRING) AS `{col}`")
644+
645+
per_view_selects.append(
646+
f"SELECT DISTINCT {', '.join(select_entities)} "
647+
f"FROM {from_expression} "
648+
f"WHERE `{timestamp_field}` BETWEEN TIMESTAMP('{start_str}') AND TIMESTAMP('{end_str}')"
649+
)
650+
651+
union_query = "\nUNION DISTINCT\n".join(per_view_selects)
652+
entity_cols = (
653+
", ".join(f"`{e}`" for e in all_entities) if all_entities else "TRUE AS _dummy"
654+
)
655+
656+
create_sql = (
657+
f"CREATE TABLE `{table_name}` AS "
658+
f"SELECT {entity_cols}, TIMESTAMP('{end_str}') AS `{event_timestamp_col}` "
659+
f"FROM ({union_query}) AS _entity_union"
660+
)
661+
662+
block_until_done(client, client.query(create_sql))
663+
664+
table = client.get_table(table_name)
665+
table.expires = _utc_now() + timedelta(minutes=30)
666+
client.update_table(table, ["expires"])
667+
668+
570669
# ------------------------------------------------------------------ #
571670
# BigQuery monitoring metrics (native)
572671
# ------------------------------------------------------------------ #

‎sdk/python/feast/infra/offline_stores/offline_utils.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,3 +429,12 @@ def format_date(val: Union[str, datetime]) -> str:
429429
filters.append(f"{dp_field} <= '{format_date(end_date)}'")
430430

431431
return " AND ".join(filters) if filters else ""
432+
433+
434+
def gather_all_entities(fv_query_contexts: List[FeatureViewQueryContext]):
435+
all_entities: List[str] = []
436+
for ctx in fv_query_contexts:
437+
for e in ctx.entities:
438+
if e not in all_entities:
439+
all_entities.append(e)
440+
return all_entities

0 commit comments

Comments
 (0)