Visitar URL original
fix: Bump decommissioned Snowflake Python UDF runtime from 3.9 to 3.10 (#6606) by zerafachris · Pull Request #6608 · feast-dev/feast · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/reference/compute-engine/snowflake.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,10 @@ batch_engine:
role: sysadmin
warehouse: demo_wh
database: FEAST
python_udf_runtime_version: "3.10"
```
{% endcode %}

## Configuration

* `python_udf_runtime_version` *(optional, default: `"3.10"`)* -- The Snowflake Python UDF `RUNTIME_VERSION` used when Feast deploys its materialization UDFs. Snowflake periodically decommissions old Python UDF runtimes (for example, the 3.9 runtime was decommissioned, requiring Feast to bump its default to 3.10 -- see [#6606](https://github.com/feast-dev/feast/issues/6606)). If Snowflake decommissions the 3.10 runtime in the future, set this field to a still-supported version (e.g. `"3.11"`) instead of waiting for a new Feast release.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import re
import shutil
from datetime import timezone
from typing import Literal, Optional, Sequence, Union
Expand All @@ -7,7 +8,7 @@
import pandas as pd
import pyarrow as pa
from colorama import Fore, Style
from pydantic import ConfigDict, Field, StrictStr
from pydantic import ConfigDict, Field, StrictStr, field_validator
from tqdm import tqdm

import feast
Expand Down Expand Up @@ -87,8 +88,41 @@ class SnowflakeComputeEngineConfig(FeastConfigBaseModel):

schema_: Optional[str] = Field("PUBLIC", alias="schema")
""" Snowflake schema name """

python_udf_runtime_version: StrictStr = "3.10"
"""
Snowflake Python UDF RUNTIME_VERSION used when Feast deploys its materialization
UDFs (see `snowflake_python_udfs_creation.sql`).

Snowflake periodically decommissions old Python UDF runtimes -- e.g. the 3.9
runtime was decommissioned, which required Feast to bump its default runtime
from 3.9 to 3.10 (see https://github.com/feast-dev/feast/issues/6606). Rather
than hardcoding a version that will eventually go stale again, this field is
user-configurable so you are not blocked on a new Feast release the next time
Snowflake deprecates a runtime.

Defaults to "3.10", matching Feast's own minimum supported Python version
(`requires-python` in `pyproject.toml`). If Snowflake decommissions the 3.10
runtime in the future, override it in your `feature_store.yaml`, e.g.:

batch_engine:
type: snowflake.engine
...
python_udf_runtime_version: "3.11"
"""

model_config = ConfigDict(populate_by_name=True, extra="allow")

@field_validator("python_udf_runtime_version")
@classmethod
def validate_python_udf_runtime_version(cls, v: str) -> str:
if not re.fullmatch(r"\d+\.\d+(\.\d+)?", v):
raise ValueError(
"python_udf_runtime_version must be a valid Python version string "
f"such as '3.10' or '3.11.2', got {v!r}"
)
return v


class SnowflakeComputeEngine(ComputeEngine):
def get_historical_features(
Expand Down Expand Up @@ -151,7 +185,11 @@ def update(
sqlCommands = sqlFile.split(";")
for command in sqlCommands:
command = command.replace("STAGE_HOLDER", f"{stage_path}")
query = command.replace("PROJECT_NAME", f"{project}")
command = command.replace("PROJECT_NAME", f"{project}")
query = command.replace(
"RUNTIME_VERSION_HOLDER",
self.repo_config.batch_engine.python_udf_runtime_version,
)
execute_snowflake_statement(conn, query)

return None
Expand Down
Original file line number Diff line number Diff line change
@@ -1,127 +1,127 @@
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_binary_to_bytes_proto(df BINARY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_binary_to_bytes_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_varchar_to_string_proto(df VARCHAR)
Comment thread
zerafachris marked this conversation as resolved.
Outdated
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_varchar_to_string_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_bytes_to_list_bytes_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_bytes_to_list_bytes_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_varchar_to_list_string_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_varchar_to_list_string_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int32_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int32_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int64_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int64_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_float_to_list_double_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_float_to_list_double_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_boolean_to_list_bool_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_boolean_to_list_bool_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_timestamp_to_list_unix_timestamp_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_number_to_int32_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int32_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_number_to_int64_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int64_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_float_to_double_proto(df DOUBLE)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_float_to_double_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_boolean_to_bool_proto(df BOOLEAN)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_boolean_to_bool_boolean_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_timestamp_to_unix_timestamp_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_timestamp_to_unix_timestamp_proto'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_serialize_entity_keys(names ARRAY, data ARRAY, types ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_serialize_entity_keys'
IMPORTS = ('@STAGE_HOLDER/feast.zip');

CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_entity_key_proto_to_string(names ARRAY, data ARRAY, types ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_entity_key_proto_to_string'
IMPORTS = ('@STAGE_HOLDER/feast.zip')
42 changes: 26 additions & 16 deletions sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,21 @@
)
from feast.value_type import ValueType

# NOTE: The `CREATE OR REPLACE FUNCTION ...` blocks below are illustrative reference
# documentation only; they are not executed as-is. The SQL Feast actually runs is
# templated from `snowflake_python_udfs_creation.sql`, which fills in placeholders
# (stage path, project name, and Python UDF RUNTIME_VERSION) at deploy time -- see
# `SnowflakeComputeEngine.update()` in `snowflake_engine.py`. The RUNTIME_VERSION
# shown here documents Feast's default (`SnowflakeComputeEngineConfig
# .python_udf_runtime_version`, currently "3.10"); if you override that config
# field, the runtime actually deployed to Snowflake will differ from what's shown
# below.

"""
CREATE OR REPLACE FUNCTION feast_snowflake_binary_to_bytes_proto(df BINARY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_binary_to_bytes_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -43,7 +53,7 @@ def feast_snowflake_binary_to_bytes_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_varchar_to_string_proto(df VARCHAR)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_varchar_to_string_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -68,7 +78,7 @@ def feast_snowflake_varchar_to_string_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_bytes_to_list_bytes_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_bytes_to_list_bytes_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand Down Expand Up @@ -96,7 +106,7 @@ def feast_snowflake_array_bytes_to_list_bytes_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_varchar_to_list_string_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_varchar_to_list_string_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -120,7 +130,7 @@ def feast_snowflake_array_varchar_to_list_string_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_number_to_list_int32_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int32_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -144,7 +154,7 @@ def feast_snowflake_array_number_to_list_int32_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_number_to_list_int64_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int64_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -168,7 +178,7 @@ def feast_snowflake_array_number_to_list_int64_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_float_to_list_double_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_float_to_list_double_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -194,7 +204,7 @@ def feast_snowflake_array_float_to_list_double_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_boolean_to_list_bool_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_boolean_to_list_bool_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -218,7 +228,7 @@ def feast_snowflake_array_boolean_to_list_bool_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_timestamp_to_list_unix_timestamp_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -244,7 +254,7 @@ def feast_snowflake_array_timestamp_to_list_unix_timestamp_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_number_to_int32_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int32_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -269,7 +279,7 @@ def feast_snowflake_number_to_int32_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_number_to_int64_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int64_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -296,7 +306,7 @@ def feast_snowflake_number_to_int64_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_float_to_double_proto(df DOUBLE)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_float_to_double_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -321,7 +331,7 @@ def feast_snowflake_float_to_double_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_boolean_to_bool_proto(df BOOLEAN)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_boolean_to_bool_boolean_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand All @@ -346,7 +356,7 @@ def feast_snowflake_boolean_to_bool_boolean_proto(df):
CREATE OR REPLACE FUNCTION feast_snowflake_timestamp_to_unix_timestamp_proto(df NUMBER)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_timestamp_to_unix_timestamp_proto'
IMPORTS = ('@feast_stage/feast.zip');
Expand Down Expand Up @@ -374,7 +384,7 @@ def feast_snowflake_timestamp_to_unix_timestamp_proto(df):
CREATE OR REPLACE FUNCTION feast_serialize_entity_keys(names ARRAY, data ARRAY, types ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_serialize_entity_keys'
IMPORTS = ('@feast_stage/feast.zip')
Expand Down Expand Up @@ -423,7 +433,7 @@ def feast_serialize_entity_keys(df):
CREATE OR REPLACE FUNCTION feast_entity_key_proto_to_string(names ARRAY, data ARRAY, types ARRAY)
RETURNS BINARY
LANGUAGE PYTHON
RUNTIME_VERSION = '3.9'
RUNTIME_VERSION = '3.10'
PACKAGES = ('protobuf', 'pandas')
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_entity_key_proto_to_string'
IMPORTS = ('@feast_stage/feast.zip')
Expand Down
Loading
Loading