Visitar URL original
fix: Decode fixed_size_list as the matching Feast list value type (#6… · feast-dev/feast@8cfe891 · GitHub
Skip to content

Commit 8cfe891

Browse files
authored
fix: Decode fixed_size_list as the matching Feast list value type (#6944)
1 parent acadb17 commit 8cfe891

2 files changed

Lines changed: 71 additions & 0 deletions

File tree

‎sdk/python/feast/type_map.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1595,8 +1595,22 @@ def _proto_value_to_value_type(proto_value: ProtoValue) -> ValueType:
15951595
return PROTO_VALUE_TO_VALUE_TYPE_MAP[proto_str]
15961596

15971597

1598+
_FIXED_SIZE_LIST_PREFIX = "fixed_size_list<item: "
1599+
1600+
15981601
def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType:
15991602
is_list = False
1603+
if pa_type_as_str.startswith(_FIXED_SIZE_LIST_PREFIX):
1604+
# A fixed-width vector is a list whose width happens to be fixed, and is
1605+
# how Arrow-native stores represent embeddings. Feast value types do not
1606+
# encode a width -- Field.vector_length carries it -- so this decodes to
1607+
# the same value type as the variable-width spelling, and is rewritten
1608+
# into that spelling so the list handling below stays single-sourced.
1609+
inner = pa_type_as_str[len(_FIXED_SIZE_LIST_PREFIX) :]
1610+
closing = inner.rfind(">")
1611+
if closing != -1:
1612+
pa_type_as_str = f"list<item: {inner[:closing]}>"
1613+
16001614
if pa_type_as_str.startswith("list<item: "):
16011615
is_list = True
16021616
inner_str = pa_type_as_str[len("list<item: ") : -1]

‎sdk/python/tests/unit/test_type_map.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2385,3 +2385,60 @@ def test_null_timestamp_int_value():
23852385
from feast.type_map import NULL_TIMESTAMP_INT_VALUE
23862386

23872387
assert NULL_TIMESTAMP_INT_VALUE == np.iinfo(np.int64).min
2388+
2389+
2390+
def test_pa_to_feast_value_type_fixed_size_list():
2391+
"""A fixed-width vector decodes to the same list type as a variable-width one.
2392+
2393+
``fixed_size_list`` is how Arrow-native stores spell an embedding column, and
2394+
is the type ``Field.vector_length`` validation reads a width from. Feast value
2395+
types carry no width, so the two spellings must agree.
2396+
"""
2397+
assert pa_to_feast_value_type("fixed_size_list<item: float>[8]") == (
2398+
ValueType.FLOAT_LIST
2399+
)
2400+
assert pa_to_feast_value_type("fixed_size_list<item: double>[1536]") == (
2401+
ValueType.DOUBLE_LIST
2402+
)
2403+
assert pa_to_feast_value_type("fixed_size_list<item: int64>[4]") == (
2404+
ValueType.INT64_LIST
2405+
)
2406+
assert pa_to_feast_value_type("fixed_size_list<item: int32>[4]") == (
2407+
ValueType.INT32_LIST
2408+
)
2409+
assert pa_to_feast_value_type("fixed_size_list<item: string>[2]") == (
2410+
ValueType.STRING_LIST
2411+
)
2412+
assert pa_to_feast_value_type("fixed_size_list<item: bool>[2]") == (
2413+
ValueType.BOOL_LIST
2414+
)
2415+
2416+
2417+
def test_pa_to_feast_value_type_fixed_size_list_matches_variable_width():
2418+
for fixed, variable in [
2419+
("fixed_size_list<item: float>[8]", "list<item: float>"),
2420+
("fixed_size_list<item: double>[1536]", "list<item: double>"),
2421+
("fixed_size_list<item: int64>[4]", "list<item: int64>"),
2422+
("fixed_size_list<item: bool>[2]", "list<item: bool>"),
2423+
]:
2424+
assert pa_to_feast_value_type(fixed) == pa_to_feast_value_type(variable)
2425+
2426+
2427+
def test_pa_to_feast_value_type_nested_fixed_size_list():
2428+
assert pa_to_feast_value_type("fixed_size_list<item: list<item: float>>[8]") == (
2429+
ValueType.VALUE_LIST
2430+
)
2431+
2432+
2433+
def test_pa_to_feast_value_type_fixed_size_list_width_is_not_encoded():
2434+
"""The width is carried by Field.vector_length, not by the value type."""
2435+
assert pa_to_feast_value_type("fixed_size_list<item: float>[8]") == (
2436+
pa_to_feast_value_type("fixed_size_list<item: float>[1536]")
2437+
)
2438+
2439+
2440+
def test_pa_to_feast_value_type_decodes_a_real_arrow_vector_type():
2441+
"""Guard the parser against the exact string PyArrow produces."""
2442+
pa_type = pyarrow.list_(pyarrow.float32(), 8)
2443+
assert str(pa_type) == "fixed_size_list<item: float>[8]"
2444+
assert pa_to_feast_value_type(str(pa_type)) == ValueType.FLOAT_LIST

0 commit comments

Comments
 (0)