Visitar URL original
feat: Add a non-JVM read path for Lance data sources by haoxu0 · Pull Request #6943 · feast-dev/feast · GitHub
Skip to content

feat: Add a non-JVM read path for Lance data sources #6943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ntkathole merged 2 commits into from
Oct 7, 2026
Merged

feat: Add a non-JVM read path for Lance data sources #6943

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e7e2adf
feat: Add a non-JVM read path for Lance data sources
haoxu0 Oct 5, 2026
8b8d17c
fix: Address Lance read review feedback
haoxu0 Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
fix: Address Lance read review feedback 
Signed-off-by: HaoXuAI <sduxuhao@gmail.com>
  • Loading branch information
@haoxu0 @ntkathole
haoxu0 authored and ntkathole committed Oct 7, 2026
commit 8b8d17c6089e9ac82d3ee94c8198cec4e2f1b762
8 changes: 3 additions & 5 deletions sdk/python/feast/infra/data_sources/contrib/lance/lance_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import json
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple

from feast.credentials import ConnectionRef
from feast.data_source import DataSource
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
from feast.repo_config import RepoConfig
Expand Down Expand Up @@ -126,7 +125,6 @@ def __init__(
description: Optional[str] = "",
tags: Optional[Dict[str, str]] = None,
owner: Optional[str] = "",
connection_ref: Optional[ConnectionRef] = None,
):
if table_format is not None and not isinstance(table_format, LanceFormat):
raise ValueError(
Expand Down Expand Up @@ -162,7 +160,6 @@ def __init__(
description=description,
tags=tags,
owner=owner,
connection_ref=connection_ref,
)
self.uri = uri
self.lance_table = table
Expand Down Expand Up @@ -282,9 +279,11 @@ def to_arrow(
of the result.
"""
dataset = self.get_dataset()
if feature_view is not None:
_validate_against_feature_view(self, dataset.schema, feature_view)

table = dataset.to_table(columns=columns)
if feature_view is not None:
_validate_against_feature_view(self, table.schema, feature_view)
from feast.utils import _validate_vector_field_lengths

_validate_vector_field_lengths(table, feature_view)
Expand Down Expand Up @@ -395,7 +394,6 @@ def from_proto(data_source: DataSourceProto) -> Any:
description=data_source.description,
tags=tags,
owner=data_source.owner,
connection_ref=ConnectionRef.from_tags(tags),
)


Expand Down
20 changes: 20 additions & 0 deletions sdk/python/tests/unit/infra/data_sources/test_lance_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import pyarrow as pa
import pytest

from feast.credentials import ConnectionRef
from feast.entity import Entity
from feast.feature_view import FeatureView
from feast.field import Field
Expand Down Expand Up @@ -135,6 +136,14 @@ def test_rejects_a_non_lance_table_format():
LanceSource(uri="/tmp/x.lance", table_format=IcebergFormat())


def test_rejects_an_unimplemented_connection_ref():
with pytest.raises(TypeError, match="unexpected keyword argument 'connection_ref'"):
LanceSource(
uri="s3://bucket/emb.lance",
connection_ref=ConnectionRef(provider="env", name="LANCE"),
)


def test_default_name_is_the_uri_for_a_path_based_source():
assert LanceSource(uri="/tmp/x.lance").name == "/tmp/x.lance"

Expand Down Expand Up @@ -259,6 +268,17 @@ def test_path_based_read_projects_columns(tmp_path):
assert source.to_arrow(columns=["driver_id"]).column_names == ["driver_id"]


def test_projection_validates_the_full_source_schema(tmp_path):
uri = str(tmp_path / "emb.lance")
_write(uri, _driver_stats())
source = LanceSource(uri=uri, timestamp_field="event_timestamp")
feature_view = _feature_view(source)

table = source.to_arrow(columns=["driver_id"], feature_view=feature_view)

assert table.column_names == ["driver_id"]


def test_path_based_schema_inference(tmp_path):
uri = str(tmp_path / "emb.lance")
_write(uri, _driver_stats())
Expand Down
Loading