feat: add plain-C scan ABI crate over Arrow C Data/Stream

Introduce datafusion-scan-ffi: a cdylib exposing a DataFusion TableProvider scan through extern "C" entrypoints that speak only C primitives and the standard Arrow C Data/Stream interface (ArrowSchema/ArrowArrayStream). No JVM/JNI dependency, so the surface is consumable from Java (via a thin shim or FFM), Python, Go, or Rust, and is a candidate to live closer to DataFusion proper. This is the JNI-free reshaping of PR #103's scan logic per review feedback on PR #104: providers are compiled in and registered by name (approach A), filters cross as datafusion.LogicalExprNode protobufs (shared vocabulary with datafusion-ffi/Comet), and each scanned partition is handed back as a zero-copy FFI_ArrowArrayStream. - abi.rs: df_scan_{schema,create,partition_count,execute_partition, execute,close}, df_error_free, df_scan_abi_version - scan.rs: build -> register -> project -> filter -> plan core - registry.rs: name-keyed provider builders - reader.rs: SendableRecordBatchStream -> panic-safe RecordBatchReader - include/datafusion_scan.h: the C header - tests/roundtrip.rs: drives the ABI and re-imports the stream via the Arrow C Stream interface, no JVM involved (6 tests) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
timsaucer · Jun 12, 2026 · Jun 12, 2026 · Jun 12, 2026 · Jun 12, 2026 · Jun 12, 2026
commit e87fd64d46d05dd9de1d34b60fedf59246e21aab
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,6 +20,7 @@ resolver = "2"
 members = [
     "native",
     "native-common",
+    "native-ffi",
 ]
 
 # Shared package metadata so every crate moves in lock step. Members inherit

diff --git a/native-ffi/Cargo.toml b/native-ffi/Cargo.toml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-scan-ffi"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+# Not published yet; this is the in-tree home of the plain-C scan ABI while it
+# stabilizes. The intent is for this surface to eventually live in DataFusion
+# proper (it has no JVM/JNI dependency), so keep it free of anything
+# Java-specific.
+publish = false
+
+[lib]
+# `cdylib`  -> the shippable plain-C shared library (`libdatafusion_scan_ffi`).
+# `rlib`    -> lets a downstream cdylib statically link this crate, register
+#              its own providers, and re-export the `df_scan_*` symbols; also
+#              gives `cargo test` a Rust harness that round-trips the ABI with
+#              no JVM in sight.
+crate-type = ["cdylib", "rlib"]
+
+[features]
+# A built-in in-memory provider builder registered under `datafusion.memory`,
+# used by the round-trip tests and handy as a reference builder. Off by default
+# so a production cdylib only carries the providers it registers itself.
+demo-providers = []
+
+[dependencies]
+# The arrow C Data / C Stream interface types are the entire data plane of this
+# ABI. `ffi` pulls in both `arrow::ffi` (FFI_ArrowSchema/Array) and
+# `arrow::ffi_stream` (FFI_ArrowArrayStream). Same crate+version DataFusion
+# links, so the types unify.
+arrow = { workspace = true }
+datafusion = { workspace = true }
+# Pushed filters arrive as serialized `datafusion.LogicalExprNode` protobufs --
+# the same vocabulary `datafusion-ffi` already uses, so the encoder is shared
+# with any future Comet path.
+datafusion-proto = { workspace = true }
+futures = { workspace = true }
+prost = { workspace = true }
+tokio = { workspace = true }
+
+[dev-dependencies]
+# Round-trip tests import the produced FFI_ArrowArrayStream back into Rust via
+# the same C Stream interface a Java/Python/Go consumer would use.
+datafusion-scan-ffi = { path = ".", features = ["demo-providers"] }
diff --git a/native-ffi/include/datafusion_scan.h b/native-ffi/include/datafusion_scan.h
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Plain-C scan ABI over the Arrow C Data / C Stream interface.
+//
+// The only "rich" types crossing this boundary are the standard Arrow C
+// structs `ArrowSchema` and `ArrowArrayStream` (from Arrow's abi.h), which any
+// Arrow implementation can produce/consume. Everything else is C primitives
+// and borrowed (ptr, len) views. No JVM/JNI types appear here, by design.
+
+#ifndef DATAFUSION_SCAN_H
+#define DATAFUSION_SCAN_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "arrow/c/abi.h"  // struct ArrowSchema, struct ArrowArrayStream
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// --- Status codes ----------------------------------------------------------
+// 0 on success; nonzero classifies the failure. On error the call also writes
+// a malloc'd, NUL-terminated message to *out_err (free with df_error_free).
+typedef enum {
+  DF_OK = 0,
+  DF_INVALID_ARGUMENT = 1,
+  DF_UNKNOWN_PROVIDER = 2,
+  DF_PROVIDER_BUILD = 3,
+  DF_PLANNING = 4,
+  DF_EXECUTION = 5,
+  DF_PANIC = 6,
+  DF_INTERNAL = 7
+} DfStatus;
+
+// --- Borrowed input views (caller owns the memory) -------------------------
+typedef struct {
+  const uint8_t* ptr;  // UTF-8, not NUL-terminated; may be null if len == 0
+  size_t len;
+} DfStr;
+
+typedef struct {
+  const uint8_t* ptr;  // may be null if len == 0
+  size_t len;
+} DfBytes;
+
+typedef struct {
+  DfStr key;
+  DfStr value;
+} DfKeyValue;
+
+// Opaque planned-scan handle.
+typedef struct DfScanHandle DfScanHandle;
+
+// --- Lifecycle / versioning ------------------------------------------------
+
+// ABI major version; compare before any other call.
+uint64_t df_scan_abi_version(void);
+
+// Free a message previously written to an out_err argument (null-safe).
+void df_error_free(char* err);
+
+// --- Scan API --------------------------------------------------------------
+
+// Probe a provider's output schema into the caller-allocated out_schema.
+int32_t df_scan_schema(DfStr provider, DfBytes options, DfBytes partition,
+                       struct ArrowSchema* out_schema, char** out_err);
+
+// Plan a scan. On success writes an owned handle to *out_handle (release with
+// df_scan_close). projection is an array of column-name DfStr (empty = all);
+// filters is an array of serialized datafusion.LogicalExprNode DfBytes;
+// target_partitions / batch_size <= 0 keep DataFusion defaults.
+int32_t df_scan_create(DfStr provider, DfBytes options, DfBytes partition,
+                       int32_t target_partitions, int32_t batch_size,
+                       const DfKeyValue* config_overrides, size_t config_overrides_len,
+                       const DfStr* projection, size_t projection_len,
+                       const DfBytes* filters, size_t filters_len,
+                       DfScanHandle** out_handle, char** out_err);
+
+// Output partition count of the planned scan.
+int32_t df_scan_partition_count(const DfScanHandle* handle, int32_t* out_count,
+                                char** out_err);
+
+// Execute one partition into the caller-allocated Arrow C Stream.
+int32_t df_scan_execute_partition(const DfScanHandle* handle, int32_t partition,
+                                  struct ArrowArrayStream* out_stream, char** out_err);
+
+// Execute the whole plan as a single coalesced Arrow C Stream.
+int32_t df_scan_execute(const DfScanHandle* handle,
+                        struct ArrowArrayStream* out_stream, char** out_err);
+
+// Drop a planned scan (null-safe). Must not race an in-flight execute on the
+// same handle.
+void df_scan_close(DfScanHandle* handle);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // DATAFUSION_SCAN_H