Visitar URL original
feat: OpenLineage consumer improvements - dataset versioning, column … · feast-dev/feast@69f294e · GitHub
Skip to content

Commit 69f294e

Browse files
committed
feat: OpenLineage consumer improvements - dataset versioning, column lineage, ownership
Signed-off-by: ntkathole <nikhilkathole2683@gmail.com>
1 parent 0b621de commit 69f294e

9 files changed

Lines changed: 1949 additions & 7 deletions

File tree

‎sdk/python/feast/openlineage/consumer.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,10 @@ def get_full_lineage_graph(
337337
"schema": schema,
338338
"description": ds.get("description"),
339339
"source_type": ds.get("source_type"),
340+
"owner": ds.get("owner_name"),
341+
"owner_type": ds.get("owner_type"),
342+
"lifecycle_state": ds.get("lifecycle_state"),
343+
"current_version": ds.get("current_version"),
340344
"facets": facets,
341345
}
342346
)
@@ -377,6 +381,61 @@ def get_full_lineage_graph(
377381
"total_nodes": total_nodes,
378382
}
379383

384+
# ── Dataset versioning endpoints ──
385+
386+
@router.get("/lineage/openlineage/datasets/{namespace}/{name}/versions")
387+
def list_dataset_versions(
388+
namespace: str,
389+
name: str,
390+
limit: int = Query(50, ge=1, le=500),
391+
offset: int = Query(0, ge=0),
392+
):
393+
"""List version history for a dataset."""
394+
ns_filter = _get_namespace_filter(get_allowed_namespaces)
395+
if ns_filter is not None and namespace not in ns_filter:
396+
return {"versions": [], "total": 0}
397+
total = store.count_dataset_versions(namespace, name)
398+
versions = store.get_dataset_versions(namespace, name, limit, offset)
399+
for v in versions:
400+
v["schema"] = _safe_parse_json(v.pop("schema_json", None))
401+
v["facets"] = _safe_parse_json(v.pop("facets_json", None))
402+
return {"versions": versions, "total": total}
403+
404+
# ── Column-level lineage endpoint ──
405+
406+
@router.get("/lineage/openlineage/datasets/{namespace}/{name}/columns")
407+
def get_column_lineage(
408+
namespace: str,
409+
name: str,
410+
direction: str = Query("both", pattern="^(both|upstream|downstream)$"),
411+
):
412+
"""Get column-level lineage for a dataset.
413+
414+
Shows which input fields map to which output fields, including
415+
transformation types when available.
416+
"""
417+
ns_filter = _get_namespace_filter(get_allowed_namespaces)
418+
if ns_filter is not None and namespace not in ns_filter:
419+
return {"column_lineage": []}
420+
cl = store.get_column_lineage(namespace, name, direction)
421+
return {"column_lineage": cl}
422+
423+
# ── Execution tree endpoint ──
424+
425+
@router.get("/lineage/openlineage/runs/{run_id}/tree")
426+
def get_run_tree(run_id: str):
427+
"""Get the full execution tree (all runs sharing the same root).
428+
429+
If *run_id* is a child run, the endpoint resolves its
430+
``root_run_id`` first so the full tree is always returned.
431+
Results are filtered to the caller's allowed namespaces.
432+
"""
433+
ns_filter = _get_namespace_filter(get_allowed_namespaces)
434+
tree = store.get_run_tree(run_id)
435+
if ns_filter is not None:
436+
tree = [r for r in tree if r.get("job_namespace") in ns_filter]
437+
return {"runs": tree, "total": len(tree)}
438+
380439
# ── Run history endpoints ──
381440

382441
@router.get("/lineage/openlineage/runs")

‎sdk/python/feast/openlineage/emitter.py‎

Lines changed: 134 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -567,10 +567,13 @@ def emit_saved_dataset_lineage(
567567
"""
568568
Emit lineage for a saved dataset definition.
569569
570-
Creates a definition job with inputs derived from:
571-
- FeatureService (when feature_service_name is set)
572-
- FeatureViews (extracted from feature refs in the format "view:feat")
573-
- DataSources (matched by comparing storage location against registered sources)
570+
Emits two events:
571+
1. A **RunEvent** that captures the definition-time topology:
572+
FeatureService/FeatureView → SavedDataset job.
573+
2. A **DatasetEvent** with SymlinksDatasetFacet that links the
574+
logical SavedDataset name to its physical storage URI, plus
575+
OwnershipDatasetFacet and LifecycleStateChangeDatasetFacet
576+
when the information is available.
574577
575578
Args:
576579
saved_dataset: The SavedDataset object
@@ -675,7 +678,7 @@ def emit_saved_dataset_lineage(
675678
**self._job_kind_facets(FeastJobKind.DEFINITION, project),
676679
}
677680

678-
return self._client.emit_run_event(
681+
run_result = self._client.emit_run_event(
679682
job_name=f"saved_dataset_{saved_dataset.name}",
680683
run_id=str(uuid.uuid4()),
681684
event_type=RunState.COMPLETE,
@@ -684,12 +687,138 @@ def emit_saved_dataset_lineage(
684687
job_facets=job_facets,
685688
namespace=namespace,
686689
)
690+
691+
# Emit a DatasetEvent with symlink, ownership, and lifecycle facets
692+
self._emit_saved_dataset_metadata(
693+
saved_dataset, project, namespace, registered_data_sources
694+
)
695+
696+
return run_result
687697
except Exception as e:
688698
logger.error(
689699
f"Error emitting saved dataset lineage for {saved_dataset.name}: {e}"
690700
)
691701
return False
692702

703+
def _emit_saved_dataset_metadata(
704+
self,
705+
saved_dataset: Any,
706+
project: str,
707+
namespace: str,
708+
registered_data_sources: Optional[List[Any]] = None,
709+
):
710+
"""Emit a DatasetEvent for a SavedDataset with OL-standard facets.
711+
712+
Includes:
713+
- SymlinksDatasetFacet linking logical name to physical storage URI
714+
- OwnershipDatasetFacet from tags (``owner`` key) or the object owner
715+
- LifecycleStateChangeDatasetFacet (CREATE)
716+
- SchemaDatasetFacet from features
717+
"""
718+
try:
719+
from openlineage.client.facet_v2 import (
720+
lifecycle_state_change_dataset,
721+
ownership_dataset,
722+
schema_dataset,
723+
symlinks_dataset,
724+
)
725+
except ImportError:
726+
logger.debug(
727+
"openlineage facet_v2 modules not available; "
728+
"skipping DatasetEvent for SavedDataset"
729+
)
730+
return
731+
732+
ds_facets: Dict[str, Any] = {}
733+
734+
# SymlinksDatasetFacet: link logical name → physical storage URI
735+
storage_uri = self._get_saved_dataset_storage_uri(saved_dataset)
736+
if storage_uri:
737+
ds_facets["symlinks"] = symlinks_dataset.SymlinksDatasetFacet(
738+
identifiers=[
739+
symlinks_dataset.Identifier(
740+
namespace=namespace,
741+
name=storage_uri,
742+
type="TABLE",
743+
)
744+
]
745+
)
746+
747+
# OwnershipDatasetFacet
748+
owner = None
749+
if hasattr(saved_dataset, "tags") and saved_dataset.tags:
750+
owner = saved_dataset.tags.get("owner")
751+
if not owner and hasattr(saved_dataset, "owner") and saved_dataset.owner:
752+
owner = saved_dataset.owner
753+
if owner:
754+
ds_facets["ownership"] = ownership_dataset.OwnershipDatasetFacet(
755+
owners=[
756+
ownership_dataset.Owner(
757+
name=owner,
758+
type="FEAST_TAG",
759+
)
760+
]
761+
)
762+
763+
# LifecycleStateChangeDatasetFacet
764+
ds_facets["lifecycleStateChange"] = (
765+
lifecycle_state_change_dataset.LifecycleStateChangeDatasetFacet(
766+
lifecycleStateChange=lifecycle_state_change_dataset.LifecycleStateChange.CREATE,
767+
)
768+
)
769+
770+
# SchemaDatasetFacet
771+
if saved_dataset.features:
772+
ds_facets["schema"] = schema_dataset.SchemaDatasetFacet(
773+
fields=[
774+
schema_dataset.SchemaDatasetFacetFields(name=f, type="UNKNOWN")
775+
for f in saved_dataset.features
776+
]
777+
)
778+
779+
try:
780+
self._client.emit_dataset_event(
781+
dataset_name=saved_dataset.name,
782+
namespace=namespace,
783+
facets=ds_facets,
784+
)
785+
except Exception as e:
786+
logger.warning(
787+
f"Failed to emit DatasetEvent for SavedDataset "
788+
f"{saved_dataset.name}: {e}"
789+
)
790+
791+
@staticmethod
792+
def _get_saved_dataset_storage_uri(saved_dataset: Any) -> Optional[str]:
793+
"""Extract a physical storage URI from a SavedDataset's storage."""
794+
if not hasattr(saved_dataset, "storage") or not saved_dataset.storage:
795+
return None
796+
try:
797+
storage_proto = saved_dataset.storage.to_proto()
798+
except Exception:
799+
return None
800+
801+
if hasattr(storage_proto, "file_url") and storage_proto.file_url:
802+
return storage_proto.file_url
803+
if hasattr(storage_proto, "bigquery_path") and storage_proto.bigquery_path:
804+
return storage_proto.bigquery_path
805+
if hasattr(storage_proto, "redshift_path") and storage_proto.redshift_path:
806+
return storage_proto.redshift_path
807+
if hasattr(storage_proto, "s3_path") and storage_proto.s3_path:
808+
return storage_proto.s3_path
809+
if hasattr(storage_proto, "trino_path") and storage_proto.trino_path:
810+
return storage_proto.trino_path
811+
812+
for attr_name in dir(storage_proto):
813+
if attr_name.startswith("_"):
814+
continue
815+
if "path" in attr_name or "url" in attr_name or "uri" in attr_name:
816+
val = getattr(storage_proto, attr_name, None)
817+
if val and isinstance(val, str):
818+
return val
819+
820+
return None
821+
693822
def emit_materialize_start(
694823
self,
695824
feature_views: List["FeatureView"],

0 commit comments

Comments
 (0)