Visitar URL original
feat: Add registry list feature views by updated since (#6092) (#6093) · feast-dev/feast@006c606 · GitHub
Skip to content

Commit 006c606

Browse files
nquinn408nquinn408
authored
feat: Add registry list feature views by updated since (#6092) (#6093)
1 parent 43263a6 commit 006c606

17 files changed

Lines changed: 669 additions & 139 deletions

File tree

‎docs/reference/feature-servers/registry-server.md‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Most endpoints support these common query parameters:
214214
- `feature` (optional): Filter feature views by feature name
215215
- `feature_service` (optional): Filter feature views by feature service name
216216
- `data_source` (optional): Filter feature views by data source name
217+
- `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`)
217218
- `page` (optional): Page number for pagination
218219
- `limit` (optional): Number of items per page
219220
- `sort_by` (optional): Field to sort by
@@ -223,27 +224,31 @@ Most endpoints support these common query parameters:
223224
# Basic list
224225
curl -H "Authorization: Bearer <token>" \
225226
"http://localhost:6572/api/v1/feature_views?project=my_project"
226-
227+
227228
# With pagination and relationships
228229
curl -H "Authorization: Bearer <token>" \
229230
"http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name"
230-
231+
231232
# Filter by entity
232233
curl -H "Authorization: Bearer <token>" \
233234
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user"
234-
235+
235236
# Filter by feature
236237
curl -H "Authorization: Bearer <token>" \
237238
"http://localhost:6572/api/v1/feature_views?project=my_project&feature=age"
238-
239+
239240
# Filter by data source
240241
curl -H "Authorization: Bearer <token>" \
241242
"http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source"
242-
243+
243244
# Filter by feature service
244245
curl -H "Authorization: Bearer <token>" \
245246
"http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service"
246-
247+
248+
# Filter by last-updated timestamp
249+
curl -H "Authorization: Bearer <token>" \
250+
"http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z"
251+
247252
# Multiple filters combined
248253
curl -H "Authorization: Bearer <token>" \
249254
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age"

‎protos/feast/registry/RegistryServer.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ message ListAllFeatureViewsRequest {
291291
string data_source = 7;
292292
PaginationParams pagination = 8;
293293
SortingParams sorting = 9;
294+
google.protobuf.Timestamp updated_since = 10;
294295
}
295296

296297
message ListAllFeatureViewsResponse {

‎sdk/python/feast/api/registry/rest/feature_views.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import logging
2+
from datetime import timezone
23
from typing import Dict, List, Optional
34

4-
from fastapi import APIRouter, Depends, Query
5+
from fastapi import APIRouter, Depends, HTTPException, Query
56
from fastapi.responses import JSONResponse
7+
from google.protobuf import timestamp_pb2
68
from google.protobuf.duration_pb2 import Duration
79
from pydantic import BaseModel
810

@@ -266,10 +268,34 @@ def list_all_feature_views(
266268
data_source: str = Query(
267269
None, description="Filter feature views by data source name"
268270
),
271+
updated_since: Optional[str] = Query(
272+
None,
273+
description="Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. 2024-01-01T00:00:00Z)",
274+
),
269275
tags: Dict[str, str] = Depends(parse_tags),
270276
pagination_params: dict = Depends(get_pagination_params),
271277
sorting_params: dict = Depends(get_sorting_params),
272278
):
279+
updated_since_proto = None
280+
if updated_since is not None:
281+
from datetime import datetime
282+
283+
try:
284+
dt = datetime.fromisoformat(updated_since.replace("Z", "+00:00"))
285+
except ValueError:
286+
raise HTTPException(
287+
status_code=400,
288+
detail=(
289+
f"Invalid 'updated_since' value '{updated_since}'; expected an "
290+
"ISO-8601 timestamp (e.g. 2024-01-01T00:00:00Z)."
291+
),
292+
)
293+
if dt.tzinfo is None:
294+
dt = dt.replace(tzinfo=timezone.utc)
295+
ts = timestamp_pb2.Timestamp()
296+
ts.FromDatetime(dt.astimezone(timezone.utc))
297+
updated_since_proto = ts
298+
273299
req = RegistryServer_pb2.ListAllFeatureViewsRequest(
274300
project=project,
275301
allow_cache=allow_cache,
@@ -280,6 +306,7 @@ def list_all_feature_views(
280306
data_source=data_source,
281307
pagination=create_grpc_pagination_params(pagination_params),
282308
sorting=create_grpc_sorting_params(sorting_params),
309+
updated_since=updated_since_proto,
283310
)
284311
response = grpc_call(grpc_handler.ListAllFeatureViews, req)
285312
any_feature_views = response.get("featureViews", [])

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,7 @@ def list_all_feature_views(
567567
allow_cache: bool = False,
568568
tags: Optional[dict[str, str]] = None,
569569
skip_udf: bool = False,
570+
updated_since: Optional[datetime] = None,
570571
) -> List[BaseFeatureView]:
571572
"""
572573
Retrieve a list of feature views of all types from the registry
@@ -576,6 +577,7 @@ def list_all_feature_views(
576577
project: Filter feature views based on project name
577578
tags: Filter by tags
578579
skip_udf: Skip deserializing UDFs (for metadata-only operations)
580+
updated_since: Only return feature views updated at or after this timestamp
579581
580582
Returns:
581583
List of feature views

‎sdk/python/feast/infra/registry/caching_registry.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import threading
44
import warnings
55
from abc import abstractmethod
6-
from datetime import timedelta
6+
from datetime import datetime, timedelta
77
from threading import Lock
88
from typing import Any, Dict, List, Optional
99

@@ -23,7 +23,7 @@
2323
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
2424
from feast.saved_dataset import SavedDataset, ValidationReference
2525
from feast.stream_feature_view import StreamFeatureView
26-
from feast.utils import _utc_now
26+
from feast.utils import _utc_now, to_naive_utc
2727

2828
logger = logging.getLogger(__name__)
2929

@@ -122,7 +122,11 @@ def get_any_feature_view(
122122

123123
@abstractmethod
124124
def _list_all_feature_views(
125-
self, project: str, tags: Optional[dict[str, str]], **kwargs: Any
125+
self,
126+
project: str,
127+
tags: Optional[dict[str, str]],
128+
updated_since: Optional[datetime] = None,
129+
**kwargs: Any,
126130
) -> List[BaseFeatureView]:
127131
pass
128132

@@ -132,13 +136,27 @@ def list_all_feature_views(
132136
allow_cache: bool = False,
133137
tags: Optional[dict[str, str]] = None,
134138
skip_udf: bool = False,
139+
updated_since: Optional[datetime] = None,
135140
) -> List[BaseFeatureView]:
136141
if allow_cache:
137142
self._refresh_cached_registry_if_necessary()
138-
return proto_registry_utils.list_all_feature_views(
143+
feature_views = proto_registry_utils.list_all_feature_views(
139144
self.cached_registry_proto, project, tags, skip_udf=skip_udf
140145
)
141-
return self._list_all_feature_views(project, tags, skip_udf=skip_udf)
146+
if updated_since is not None:
147+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
148+
cutoff = to_naive_utc(updated_since)
149+
feature_views = [
150+
fv
151+
for fv in feature_views
152+
if fv.last_updated_timestamp is not None
153+
and fv.last_updated_timestamp >= cutoff
154+
]
155+
else:
156+
feature_views = self._list_all_feature_views(
157+
project, tags, updated_since=updated_since, skip_udf=skip_udf
158+
)
159+
return feature_views
142160

143161
@abstractmethod
144162
def _get_feature_view(self, name: str, project: str) -> FeatureView:

‎sdk/python/feast/infra/registry/registry.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
from feast.repo_contents import RepoContents
5959
from feast.saved_dataset import SavedDataset, ValidationReference
6060
from feast.stream_feature_view import StreamFeatureView
61-
from feast.utils import _utc_now
61+
from feast.utils import _utc_now, to_naive_utc
6262
from feast.version_utils import (
6363
generate_version_id,
6464
parse_version,
@@ -1083,13 +1083,24 @@ def list_all_feature_views(
10831083
allow_cache: bool = False,
10841084
tags: Optional[dict[str, str]] = None,
10851085
skip_udf: bool = False,
1086+
updated_since: Optional[datetime] = None,
10861087
) -> List[BaseFeatureView]:
10871088
registry_proto = self._get_registry_proto(
10881089
project=project, allow_cache=allow_cache
10891090
)
1090-
return proto_registry_utils.list_all_feature_views(
1091+
feature_views = proto_registry_utils.list_all_feature_views(
10911092
registry_proto, project, tags, skip_udf=skip_udf
10921093
)
1094+
if updated_since is not None:
1095+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
1096+
cutoff = to_naive_utc(updated_since)
1097+
feature_views = [
1098+
fv
1099+
for fv in feature_views
1100+
if fv.last_updated_timestamp is not None
1101+
and fv.last_updated_timestamp >= cutoff
1102+
]
1103+
return feature_views
10931104

10941105
def get_any_feature_view(
10951106
self, name: str, project: str, allow_cache: bool = False

‎sdk/python/feast/infra/registry/remote.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,9 +403,18 @@ def list_all_feature_views(
403403
allow_cache: bool = False,
404404
tags: Optional[dict[str, str]] = None,
405405
skip_udf: bool = False,
406+
updated_since: Optional[datetime] = None,
406407
) -> List[BaseFeatureView]:
408+
updated_since_proto = None
409+
if updated_since is not None:
410+
ts = Timestamp()
411+
ts.FromDatetime(updated_since)
412+
updated_since_proto = ts
407413
request = RegistryServer_pb2.ListAllFeatureViewsRequest(
408-
project=project, allow_cache=allow_cache, tags=tags
414+
project=project,
415+
allow_cache=allow_cache,
416+
tags=tags,
417+
updated_since=updated_since_proto,
409418
)
410419

411420
response: RegistryServer_pb2.ListAllFeatureViewsResponse = (

‎sdk/python/feast/infra/registry/snowflake.py‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
from feast.repo_config import RegistryConfig
6363
from feast.saved_dataset import SavedDataset, ValidationReference
6464
from feast.stream_feature_view import StreamFeatureView
65-
from feast.utils import _utc_now, has_all_tags
65+
from feast.utils import _utc_now, has_all_tags, to_naive_utc
6666

6767
logger = logging.getLogger(__name__)
6868

@@ -656,14 +656,24 @@ def list_all_feature_views(
656656
allow_cache: bool = False,
657657
tags: Optional[dict[str, str]] = None,
658658
skip_udf: bool = False,
659+
updated_since: Optional[datetime] = None,
659660
) -> List[BaseFeatureView]:
660661
if allow_cache:
661662
registry_proto = self._refresh_cached_registry_if_necessary()
662-
return proto_registry_utils.list_all_feature_views(
663+
feature_views = proto_registry_utils.list_all_feature_views(
663664
registry_proto, project, tags, skip_udf=skip_udf
664665
)
666+
if updated_since is not None:
667+
cutoff = to_naive_utc(updated_since)
668+
feature_views = [
669+
fv
670+
for fv in feature_views
671+
if fv.last_updated_timestamp is not None
672+
and fv.last_updated_timestamp >= cutoff
673+
]
674+
return feature_views
665675

666-
return (
676+
feature_views = (
667677
cast(
668678
list[BaseFeatureView],
669679
self.list_feature_views(project, allow_cache, tags, skip_udf=skip_udf),
@@ -686,6 +696,17 @@ def list_all_feature_views(
686696
)
687697
)
688698

699+
if updated_since is not None:
700+
cutoff = to_naive_utc(updated_since)
701+
feature_views = [
702+
fv
703+
for fv in feature_views
704+
if fv.last_updated_timestamp is not None
705+
and fv.last_updated_timestamp >= cutoff
706+
]
707+
708+
return feature_views
709+
689710
def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
690711
infra_object = self._get_object(
691712
"MANAGED_INFRA",

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -485,26 +485,36 @@ def _get_any_feature_view(self, name: str, project: str) -> BaseFeatureView:
485485
return fv
486486

487487
def _list_all_feature_views(
488-
self, project: str, tags: Optional[dict[str, str]], **kwargs
488+
self,
489+
project: str,
490+
tags: Optional[dict[str, str]],
491+
updated_since: Optional[datetime] = None,
492+
**kwargs,
489493
) -> List[BaseFeatureView]:
490494
return (
491495
cast(
492496
list[BaseFeatureView],
493-
self._list_feature_views(project=project, tags=tags, **kwargs),
497+
self._list_feature_views(
498+
project=project, tags=tags, updated_since=updated_since, **kwargs
499+
),
494500
)
495501
+ cast(
496502
list[BaseFeatureView],
497-
self._list_stream_feature_views(project=project, tags=tags, **kwargs),
503+
self._list_stream_feature_views(
504+
project=project, tags=tags, updated_since=updated_since, **kwargs
505+
),
498506
)
499507
+ cast(
500508
list[BaseFeatureView],
501509
self._list_on_demand_feature_views(
502-
project=project, tags=tags, **kwargs
510+
project=project, tags=tags, updated_since=updated_since, **kwargs
503511
),
504512
)
505513
+ cast(
506514
list[BaseFeatureView],
507-
self._list_label_views(project=project, tags=tags, **kwargs),
515+
self._list_label_views(
516+
project=project, tags=tags, updated_since=updated_since, **kwargs
517+
),
508518
)
509519
)
510520

@@ -1597,6 +1607,7 @@ def _list_objects(
15971607
tags: Optional[dict[str, str]] = None,
15981608
proto_only: bool = False,
15991609
skip_udf: bool = False,
1610+
updated_since: Optional[datetime] = None,
16001611
):
16011612
"""
16021613
Args:
@@ -1618,6 +1629,17 @@ def _list_objects(
16181629

16191630
with self.read_engine.begin() as conn:
16201631
stmt = select(table).where(table.c.project_id == project)
1632+
if updated_since is not None:
1633+
# Ensure naive datetimes are treated as UTC, consistent with
1634+
# the Python-side filters that compare against offset-naive UTC
1635+
# last_updated_timestamp values from protobuf.
1636+
if updated_since.tzinfo is None:
1637+
updated_since_utc = updated_since.replace(tzinfo=timezone.utc)
1638+
else:
1639+
updated_since_utc = updated_since.astimezone(timezone.utc)
1640+
stmt = stmt.where(
1641+
table.c.last_updated_timestamp >= int(updated_since_utc.timestamp())
1642+
)
16211643
rows = conn.execute(stmt).all()
16221644
if rows:
16231645
objects = []

‎sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py‎

Lines changed: 112 additions & 112 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)