Visitar URL original
feat: Clickhouse offline store by iamhatesz · Pull Request #4725 · feast-dev/feast · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
aa2c839
Clickhouse offline store - initial working version
iamhatesz Oct 31, 2024
1d0417e
Remove untested `pull_all_from_table_or_query`
iamhatesz Oct 31, 2024
3af1889
Reorder functions
iamhatesz Oct 31, 2024
5d7f173
Remove commented line
iamhatesz Oct 31, 2024
9d9d24a
Fix frozen mypy errors
iamhatesz Oct 31, 2024
8207c44
mypy fixes; remove online source creator
iamhatesz Oct 31, 2024
7aa45d6
Remove commented code
iamhatesz Oct 31, 2024
803752f
Added docs
iamhatesz Oct 31, 2024
7fffc15
Python 3.9 deps
iamhatesz Oct 31, 2024
6e35db8
Python 3.10 deps
iamhatesz Oct 31, 2024
ca9d795
Python 3.11 deps (updated)
iamhatesz Oct 31, 2024
f8a8356
Remove unused ClickhouseOnlineStoreConfig
iamhatesz Oct 31, 2024
a8aa40b
Merge remote-tracking branch 'origin/master' into feast-clickhouse-of…
iamhatesz Feb 11, 2025
7ce7db1
Regenerate requirements.txt files
iamhatesz Feb 11, 2025
bcb90ca
Lint & format fixes
iamhatesz Feb 11, 2025
27a1e6f
Merge remote-tracking branch 'origin/master' into feast-clickhouse-of…
iamhatesz Mar 6, 2025
80bf5d7
Regenerate requirements.txt files
iamhatesz Mar 6, 2025
53839a8
Add clickhouse to pyproject.toml
iamhatesz Mar 7, 2025
9872944
Fix dependencies
iamhatesz Mar 7, 2025
e8e65b1
Simplify names
iamhatesz Mar 7, 2025
c0f19b2
Skip problematic Clickhouse tests
iamhatesz Mar 7, 2025
2d69427
format & lint
iamhatesz Mar 7, 2025
d5c5083
Merge remote-tracking branch 'origin/master' into feast-clickhouse-of…
iamhatesz Mar 12, 2025
3258903
Post-merge `make lock-python-dependencies-all`
iamhatesz Mar 12, 2025
df6ad54
Pin torch to 2.2.2
iamhatesz Mar 12, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
mypy fixes; remove online source creator
Signed-off-by: Tomasz Wrona <tomasz@cast.ai>
  • Loading branch information
iamhatesz committed Oct 31, 2024
commit 8207c44cd379eed772382b2d02601366fa7f86a0
Original file line number Diff line number Diff line change
Expand Up @@ -382,21 +382,22 @@ def _arrow_to_ch_timestamp_type(t_str: str) -> str:
unit = unit.strip()
precision = _ARROW_PRECISION_TO_CH_PRECISION[unit]

tz: Optional[str] = None
if len(rest):
tz: str = rest[0]
tz = rest[0]
tz = (
tz.strip()
.removeprefix("tz=")
.translate(
str.maketrans(
{
{ # type: ignore[arg-type]
"'": None,
'"': None,
}
)
)
)
else:
tz = None

if precision > 0:
if tz is not None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,9 +173,9 @@ def to_data_source(self) -> DataSource:
return ClickhouseSource(table=self.clickhouse_options._table)


def ch_type_to_feast_value_type(type_: str) -> ValueType:
type_obj = get_from_name(type_)
type_ = type(type_obj)
def ch_type_to_feast_value_type(type_str: str) -> ValueType:
type_obj = get_from_name(type_str)
type_cls = type(type_obj)
container_type = None
if isinstance(type_obj, Array):
container_type = Array
Expand All @@ -201,9 +201,9 @@ def ch_type_to_feast_value_type(type_: str) -> ValueType:
(Array, DateTime): ValueType.UNIX_TIMESTAMP_LIST,
(Array, DateTime64): ValueType.UNIX_TIMESTAMP_LIST,
}
value = type_map.get((container_type, type_), ValueType.UNKNOWN)
value = type_map.get((container_type, type_cls), ValueType.UNKNOWN)
if value == ValueType.UNKNOWN:
print("unknown type:", type_)
print("unknown type:", type_str)
return value


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,9 @@
ClickhouseSource,
)
from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig
from feast.infra.utils.clickhouse.connection_utils import get_client
from feast.saved_dataset import SavedDatasetStorage
from tests.integration.feature_repos.universal.data_source_creator import (
DataSourceCreator,
)
from tests.integration.feature_repos.universal.online_store_creator import (
OnlineStoreCreator,
)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -61,7 +56,7 @@ def clickhouse_container():
container.stop()


class ClickhouseDataSourceCreator(DataSourceCreator, OnlineStoreCreator):
class ClickhouseDataSourceCreator(DataSourceCreator):
def create_logged_features_destination(self) -> LoggingDestination:
return None # type: ignore

Expand Down Expand Up @@ -101,6 +96,8 @@ def create_data_source(
destination_name = self.get_prefixed_table_name(destination_name)

if self.offline_store_config:
if timestamp_field is None:
timestamp_field = "ts"
df_to_clickhouse_table(
self.offline_store_config, df, destination_name, timestamp_field
)
Expand All @@ -119,24 +116,8 @@ def create_offline_store_config(self) -> ClickhouseOfflineStoreConfig:
def get_prefixed_table_name(self, suffix: str) -> str:
return f"{self.project_name}_{suffix}"

def create_online_store(self) -> ClickhouseOnlineStoreConfig:
assert self.container

get_client(self.offline_store_config).command(
f"CREATE DATABASE {CLICKHOUSE_ONLINE_DB}"
)

return ClickhouseOnlineStoreConfig(
type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore",
host="localhost",
port=self.container.get_exposed_port(8123),
database=CLICKHOUSE_ONLINE_DB,
user=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
)

def create_saved_dataset_destination(self) -> SavedDatasetStorage:
return None
def create_saved_dataset_destination(self):
pass

def teardown(self):
pass
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from pydantic import StrictStr, ConfigDict
from pydantic import ConfigDict, StrictStr

from feast.repo_config import FeastConfigBaseModel

Expand Down