Repository navigation
Expand file tree
/
Copy pathCargo.toml
More file actions
410 lines (395 loc) · 17.1 KB
/
Copy pathCargo.toml
File metadata and controls
410 lines (395 loc) · 17.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[workspace]
members = [
"datafusion/common",
"datafusion/common-runtime",
"datafusion/catalog",
"datafusion/catalog-listing",
"datafusion/datasource",
"datafusion/datasource-arrow",
"datafusion/datasource-avro",
"datafusion/datasource-csv",
"datafusion/datasource-json",
"datafusion/datasource-parquet",
"datafusion/core",
"datafusion/expr",
"datafusion/expr-common",
"datafusion/execution",
"datafusion/ffi",
"datafusion/functions",
"datafusion/functions-aggregate",
"datafusion/functions-aggregate-common",
"datafusion/functions-table",
"datafusion/functions-nested",
"datafusion/functions-window",
"datafusion/functions-window-common",
"datafusion/optimizer",
"datafusion/physical-expr",
"datafusion/physical-expr-adapter",
"datafusion/physical-expr-common",
"datafusion/physical-optimizer",
"datafusion/pruning",
"datafusion/physical-plan",
"datafusion/proto",
"datafusion/proto-common",
"datafusion/proto-common/gen",
"datafusion/proto-models",
"datafusion/proto-models/gen",
"datafusion/session",
"datafusion/spark",
"datafusion/sql",
"datafusion/sqllogictest",
"datafusion/substrait",
"datafusion/wasmtest",
"datafusion-cli",
"datafusion-examples",
"datafusion-examples/examples/ffi/ffi_example_table_provider",
"datafusion-examples/examples/ffi/ffi_module_interface",
"datafusion-examples/examples/ffi/ffi_module_loader",
"test-utils",
"benchmarks",
"datafusion/macros",
"datafusion/doc",
"xtask",
]
exclude = ["dev/depcheck"]
resolver = "2"
[workspace.package]
authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
edition = "2024"
homepage = "https://datafusion.apache.org"
license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/apache/datafusion"
# Define Minimum Supported Rust Version (MSRV)
rust-version = "1.95.0"
# Define DataFusion version
version = "55.1.0"
[workspace.dependencies]
# We turn off default-features for some dependencies here so the workspaces which inherit them can
# selectively turn them on if needed, since we can override default-features = true (from false)
# for the inherited dependency but cannot do the reverse (override from true to false).
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
arrow = { version = "60.0.0", features = [
"prettyprint",
"chrono-tz",
] }
arrow-avro = { version = "60.0.0", default-features = false, features = [
"deflate",
"snappy",
"zstd",
"bzip2",
"xz",
] }
arrow-buffer = { version = "60.0.0", default-features = false }
arrow-data = { version = "60.0.0", default-features = false }
arrow-flight = { version = "60.0.0", features = [
"flight-sql",
] }
# Both codecs are required here to make sure that code paths like
# file-spilling have access to all compression codecs.
arrow-ipc = { version = "60.0.0", default-features = false, features = [
"lz4",
"zstd",
] }
arrow-ord = { version = "60.0.0", default-features = false }
arrow-schema = { version = "60.0.0", default-features = false }
arrow-select = { version = "60.0.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
bzip2 = "0.6.1"
chrono = { version = "0.4.45", default-features = false }
criterion = "0.8"
ctor = "1.0.7"
dashmap = "6.2.1"
datafusion = { path = "datafusion/core", version = "55.1.0", default-features = false }
datafusion-catalog = { path = "datafusion/catalog", version = "55.1.0" }
datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "55.1.0" }
datafusion-common = { path = "datafusion/common", version = "55.1.0", default-features = false }
datafusion-common-runtime = { path = "datafusion/common-runtime", version = "55.1.0" }
datafusion-datasource = { path = "datafusion/datasource", version = "55.1.0", default-features = false }
datafusion-datasource-arrow = { path = "datafusion/datasource-arrow", version = "55.1.0", default-features = false }
datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "55.1.0", default-features = false }
datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "55.1.0", default-features = false }
datafusion-datasource-json = { path = "datafusion/datasource-json", version = "55.1.0", default-features = false }
datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "55.1.0", default-features = false }
datafusion-doc = { path = "datafusion/doc", version = "55.1.0" }
datafusion-execution = { path = "datafusion/execution", version = "55.1.0", default-features = false }
datafusion-expr = { path = "datafusion/expr", version = "55.1.0", default-features = false }
datafusion-expr-common = { path = "datafusion/expr-common", version = "55.1.0" }
datafusion-ffi = { path = "datafusion/ffi", version = "55.1.0" }
datafusion-functions = { path = "datafusion/functions", version = "55.1.0" }
datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "55.1.0" }
datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "55.1.0" }
datafusion-functions-nested = { path = "datafusion/functions-nested", version = "55.1.0", default-features = false }
datafusion-functions-table = { path = "datafusion/functions-table", version = "55.1.0" }
datafusion-functions-window = { path = "datafusion/functions-window", version = "55.1.0" }
datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "55.1.0" }
datafusion-macros = { path = "datafusion/macros", version = "55.1.0" }
datafusion-optimizer = { path = "datafusion/optimizer", version = "55.1.0", default-features = false }
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "55.1.0", default-features = false }
datafusion-physical-expr-adapter = { path = "datafusion/physical-expr-adapter", version = "55.1.0", default-features = false }
datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "55.1.0", default-features = false }
datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "55.1.0" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "55.1.0" }
datafusion-proto = { path = "datafusion/proto", version = "55.1.0", default-features = false }
datafusion-proto-common = { path = "datafusion/proto-common", version = "55.1.0" }
datafusion-proto-models = { path = "datafusion/proto-models", version = "55.1.0" }
datafusion-pruning = { path = "datafusion/pruning", version = "55.1.0" }
datafusion-session = { path = "datafusion/session", version = "55.1.0" }
datafusion-spark = { path = "datafusion/spark", version = "55.1.0" }
datafusion-sql = { path = "datafusion/sql", version = "55.1.0" }
datafusion-substrait = { path = "datafusion/substrait", version = "55.1.0" }
doc-comment = "0.3"
env_logger = "0.11"
flate2 = "1.1.9"
futures = "0.3"
glob = "0.3.0"
half = { version = "2.7.0", default-features = false }
hashbrown = { version = "0.17.1" }
hex = { version = "0.4.3" }
indexmap = "2.14.0"
insta = { version = "1.47.2", features = ["glob", "filters"] }
itertools = "0.15"
itoa = "1.0"
liblzma = { version = "0.4.6", features = ["static"] }
log = "^0.4"
loom = "0.7"
memchr = "2.8.1"
num-traits = { version = "0.2" }
object_store = { version = "0.14.2", default-features = false }
parking_lot = "0.12"
parquet = { version = "60.0.0", default-features = false, features = [
"arrow",
"async",
"object_store",
] }
pbjson = { version = "0.9.0" }
pbjson-types = "0.9"
percent-encoding = "2.3"
pin-project = "1"
pin-project-lite = "^0.2.7"
# Should match arrow-flight's version of prost.
prost = "0.14.1"
rand = "0.9"
recursive = "0.1.1"
regex = "1.12"
rstest = "0.27.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "^0.11.0"
sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] }
stacker = "0.1.24"
strum = "0.28.0"
strum_macros = "0.28.0"
tempfile = "3"
testcontainers-modules = { version = "0.15" }
tokio = { version = "1.52", features = ["macros", "rt", "sync"] }
tokio-stream = "0.1"
tokio-util = "0.7"
url = "2.5.7"
uuid = "1.23"
zstd = { version = "0.14", default-features = false }
# Keep this list sorted alphabetically.
# See https://github.com/apache/datafusion/issues/18467 for the ongoing effort of
# picking useful non-default lints.
[workspace.lints.clippy]
all = { level = "warn", priority = -1 }
pedantic = { level = "warn", priority = -1 }
# https://github.com/apache/datafusion/issues/18881
allow_attributes = "warn"
as_ptr_cast_mut = "warn"
branches_sharing_code = "warn"
clear_with_drain = "warn"
coerce_container_to_any = "warn"
debug_assert_with_mut_call = "warn"
default_union_representation = "warn"
doc_include_without_cfg = "warn"
empty_enum_variants_with_brackets = "warn"
equatable_if_let = "warn"
exit = "warn"
fn_to_numeric_cast_any = "warn"
imprecise_flops = "warn"
infinite_loop = "warn"
iter_on_empty_collections = "warn"
iter_on_single_items = "warn"
large_include_file = "warn"
# Guards against stack overflows
large_stack_frames = "warn"
# Catches `"{foo}"` where the string is never actually formatted
literal_string_with_formatting_args = "warn"
mem_forget = "warn"
needless_type_cast = "warn"
negative_feature_names = "warn"
non_zero_suggestions = "warn"
nonstandard_macro_braces = "warn"
or_fun_call = "warn"
path_buf_push_overwrite = "warn"
pathbuf_init_then_push = "warn"
precedence_bits = "warn"
pub_without_shorthand = "warn"
rc_mutex = "warn"
rest_pat_in_fully_bound_structs = "warn"
# Avoids hashing the key twice
set_contains_or_insert = "warn"
single_option_map = "warn"
string_lit_as_bytes = "warn"
string_lit_chars_any = "warn"
suspicious_xor_used_as_pow = "warn"
trailing_empty_array = "warn"
trait_duplication_in_bounds = "warn"
uninhabited_references = "warn"
unnecessary_safety_doc = "warn"
unnecessary_self_imports = "warn"
unnecessary_struct_initialization = "warn"
unused_peekable = "warn"
unused_rounding = "warn"
verbose_file_reads = "warn"
wildcard_dependencies = "warn"
# Pedantic lints we opt out of, with the number of hits at the time we enabled `pedantic`.
# Some of these we should consider enabling.
assert_is_empty = "allow" # 105 hits
cast_lossless = "allow" # 361 hits
cast_possible_truncation = "allow" # 911 hits
cast_possible_wrap = "allow" # 493 hits
cast_precision_loss = "allow" # 266 hits
cast_sign_loss = "allow" # 440 hits
default_trait_access = "allow" # 221 hits
doc_markdown = "allow" # 4933 hits; needs a long `doc-valid-idents` list in `clippy.toml`
enum_glob_use = "allow" # 98 hits
explicit_into_iter_loop = "allow" # 55 hits
explicit_iter_loop = "allow" # 189 hits
float_cmp = "allow" # 8 hits; exact float comparisons are often intentional here
if_not_else = "allow" # 133 hits
ignored_unit_patterns = "allow" # 52 hits
implicit_clone = "allow" # 198 hits
implicit_hasher = "allow" # 17 hits; some sites feed arrow APIs that require the default hasher
inline_always = "allow" # 45 hits
items_after_statements = "allow" # 171 hits
manual_string_new = "allow" # 84 hits
many_single_char_names = "allow" # 12 hits; short names are idiomatic in the numeric kernels
map_unwrap_or = "allow" # 198 hits
match_bool = "allow" # 46 hits
match_same_arms = "allow" # 261 hits
match_wildcard_for_single_variants = "allow" # 132 hits
missing_errors_doc = "allow" # 1807 hits
missing_fields_in_debug = "allow" # 29 hits
missing_panics_doc = "allow" # 244 hits
must_use_candidate = "allow" # 2726 hits
needless_raw_string_hashes = "allow" # 540 hits
redundant_closure_for_method_calls = "allow" # 686 hits
redundant_else = "allow" # 48 hits
return_self_not_must_use = "allow" # 644 hits
semicolon_if_nothing_returned = "allow" # 1353 hits
similar_names = "allow" # 228 hits; too many false positives, e.g. `expr`/`exprs`
single_match_else = "allow" # 155 hits
struct_excessive_bools = "allow" # 24 hits
struct_field_names = "allow" # 14 hits
too_many_lines = "allow" # 484 hits
trivially_copy_pass_by_ref = "allow" # 74 hits
unnecessary_literal_bound = "allow" # 471 hits
unnecessary_wraps = "allow" # 427 hits
unnested_or_patterns = "allow" # 68 hits
unreadable_literal = "allow" # 502 hits
unused_self = "allow" # 69 hits
used_underscore_items = "allow" # 28 hits
wildcard_imports = "allow" # 48 hits; `use crate::prelude::*` is idiomatic
[workspace.lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = [
'cfg(datafusion_coop, values("tokio", "tokio_fallback", "per_stream"))',
"cfg(coverage)",
"cfg(coverage_nightly)",
# Enables the loom model-checked concurrency tests; see
# datafusion/physical-plan/src/joins/hash_join/probe_completion.rs.
# Namespaced rather than a bare `loom`, which tokio also reads and which
# would switch tokio itself into its loom build.
"cfg(datafusion_loom)",
] }
unused_qualifications = "deny"
# --------------------
# Compilation Profiles
# --------------------
# A Cargo profile is a preset for the compiler/linker knobs that trades off:
# - Build time: how quickly code compiles and links
# - Runtime performance: how fast the resulting binaries execute
# - Binary size: how large the executables end up
# - Debuggability: how much debug information is preserved for debugging and profiling
#
# To use a profile: `cargo [ build | run | ... ] --profile <profile-name>`
#
# Profiles available:
# - dev: default debug build; fastest to compile, slowest to run. Carries line
# tables only, so panics and backtraces still name a file and line but a
# debugger cannot inspect variables; build with `CARGO_PROFILE_DEV_DEBUG=2`
# when you need that. For everyday development; default for
# "cargo [ build | test | run ]".
# - release: fully optimized build; slowest to compile, fastest to run, smallest
# binaries. For public releases; default for "cargo [ bench | install ]".
# - release-nonlto: skips LTO, so it builds much faster while staying close to
# release performance. Useful when developing performance optimizations.
# - profiling: inherits release optimizations but retains debug info to support
# profiling tools and flamegraphs.
# - ci: derived from `dev` but disables debug info and incremental builds to keep
# CI artifacts small and reproducible.
# - ci-optimized: derived from `release` but enables debug assertions and uses
# less aggressive optimizations for faster builds. Used for long-running CI
# tasks.
#
# If you want to optimize compilation, the `compile_profile` benchmark can be useful.
# See `benchmarks/README.md` for more details.
# `dev` is the profile every `cargo build`/`cargo test` uses, so its debug info
# is generated over and over. `line-tables-only` keeps file and line numbers --
# so panics and `RUST_BACKTRACE` output stay useful -- while dropping the
# variable-level DWARF that only an interactive debugger consumes. Build with
# `CARGO_PROFILE_DEV_DEBUG=2 cargo build` when you need to step through code in a
# debugger.
[profile.dev]
debug = "line-tables-only"
[profile.release]
codegen-units = 1
lto = true
strip = true # Eliminate debug info to minimize binary size
[profile.release-nonlto]
inherits = "release"
codegen-units = 16
lto = false
strip = false # Retain debug info for flamegraphs
[profile.profiling]
inherits = "release"
debug = true
strip = false
[profile.ci-optimized]
inherits = "release"
debug-assertions = true
codegen-units = 16
lto = "thin"
[profile.ci]
inherits = "dev"
debug = false
incremental = false
# This rule applies to every package except workspace members (dependencies
# such as `arrow` and `tokio`). It disables debug info and related features on
# dependencies so their binaries stay smaller, improving cache reuse.
[profile.ci.package."*"]
debug = false
debug-assertions = false
strip = "debuginfo"
incremental = false