Visitar URL original
feat(loader): bundle native library inside the JAR by andygrove · Pull Request #77 · apache/datafusion-java · GitHub
Skip to content

feat(loader): bundle native library inside the JAR #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andygrove merged 2 commits into from
May 25, 2026
Merged

feat(loader): bundle native library inside the JAR #77

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
283e79e
feat(loader): bundle native library inside the JAR
andygrove May 21, 2026
9458b4e
fix(loader): isolate native lib extraction and add Windows build profile
andygrove May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
fix(loader): isolate native lib extraction and add Windows build profile 
Address PR #77 review feedback:

- NativeLibraryLoader extracts to a per-JVM private temp directory via
  Files.createTempDirectory (mode 0700 on POSIX, user-private on Windows)
  instead of a shared $TMPDIR/datafusion-java/<sha256>/ tree. Removes the
  size-only existence check that would have let another user plant a
  same-sized library at the predictable hash path.
- Add native-windows-amd64 Maven profile so datafusion.lib.os/arch/filename
  are defined on Windows; otherwise the process-classes copy step would
  abort the build even when the loader could fall back to System.loadLibrary.
- Map Windows x86_64 -> amd64 in Platform.detectArch to match the new
  profile and Linux convention; extend PlatformTest accordingly.
  • Loading branch information
@andygrove
andygrove committed May 25, 2026
commit 9458b4e33b426404bd30825925a4eb3a71ea124c
12 changes: 12 additions & 0 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ under the License.
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
org/apache/datafusion/windows/amd64/datafusion_jni.dll
-->
<profile>
<id>native-linux-amd64</id>
Expand Down Expand Up @@ -248,5 +249,16 @@ under the License.
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
</properties>
</profile>
<profile>
<id>native-windows-amd64</id>
<activation>
<os><family>windows</family><arch>amd64</arch></os>
</activation>
<properties>
<datafusion.lib.os>windows</datafusion.lib.os>
<datafusion.lib.arch>amd64</datafusion.lib.arch>
<datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
</properties>
</profile>
</profiles>
</project>
104 changes: 25 additions & 79 deletions core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,36 +21,28 @@

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

/**
* Loads the {@code datafusion_jni} native library on demand.
*
* <p>The loader first tries {@link System#loadLibrary(String)} so that
* operators can override the bundled library by placing a build on
* {@code java.library.path} (for example via
* {@code -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that
* fails the loader extracts the platform-specific library from the JAR
* resource tree and loads it via {@link System#load(String)}.
* <p>The loader first tries {@link System#loadLibrary(String)} so that operators can override the
* bundled library by placing a build on {@code java.library.path} (for example via {@code
* -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that fails the loader extracts the
* platform-specific library from the JAR resource tree and loads it via {@link
* System#load(String)}.
*
* <p>The bundled libraries live at the conventional path
* {@code org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}.
* Extracted files are written under
* {@code $TMPDIR/datafusion-java/&lt;sha256&gt;/} so that concurrent JVMs
* sharing a temp directory converge on the same file rather than each
* extracting their own copy.
* <p>The bundled libraries live at the conventional path {@code
* org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}. Each JVM extracts into its own
* private temporary directory created with {@link Files#createTempDirectory(String,
* java.nio.file.attribute.FileAttribute[])}, which is mode 0700 on POSIX and user-private under the
* default Windows {@code TMP}. Keeping the directory private prevents another user on the host from
* planting a same-named library for the JVM to pick up.
*/
public final class NativeLibraryLoader {

private static final String TMP_DIR_NAME = "datafusion-java";
private static final String TMP_DIR_PREFIX = "datafusion-java-";

private static volatile boolean loaded;

Expand Down Expand Up @@ -83,8 +75,11 @@ private static void loadFromClasspath() {
try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
if (check == null) {
throw new UnsatisfiedLinkError(
"No bundled datafusion_jni library for " + platform
+ " (expected classpath:" + resource + ")."
"No bundled datafusion_jni library for "
+ platform
+ " (expected classpath:"
+ resource
+ ")."
+ " Build the native crate and add it to java.library.path,"
+ " or depend on a JAR built for this platform.");
}
Expand All @@ -93,73 +88,24 @@ private static void loadFromClasspath() {
}

try {
Path extracted = extractToTempDir(resource, platform.libFileName());
Path extracted = extractToPrivateTempDir(resource, platform.libFileName());
System.load(extracted.toAbsolutePath().toString());
} catch (IOException e) {
throw linkError("Failed to extract " + resource, e);
}
}

private static Path extractToTempDir(String resource, String fileName) throws IOException {
Path tmpRoot = Files.createDirectories(
Paths.get(System.getProperty("java.io.tmpdir"), TMP_DIR_NAME));
Path staging = Files.createTempFile(tmpRoot, fileName + ".", ".part");

String hash;
try (InputStream raw = NativeLibraryLoader.class.getResourceAsStream(resource);
DigestInputStream in = new DigestInputStream(raw, sha256());
OutputStream out = Files.newOutputStream(staging)) {
in.transferTo(out);
hash = toHex(in.getMessageDigest().digest());
} catch (IOException e) {
Files.deleteIfExists(staging);
throw e;
}

Path versionedDir = Files.createDirectories(tmpRoot.resolve(hash));
Path target = versionedDir.resolve(fileName);

if (Files.exists(target) && Files.size(target) == Files.size(staging)) {
Files.deleteIfExists(staging);
return target;
}

try {
Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE);
} catch (FileAlreadyExistsException e) {
// Another JVM extracted the same content while we were writing.
// Their copy is identical (same SHA-256), so discard ours.
Files.deleteIfExists(staging);
} catch (IOException e) {
// Atomic move not supported on this filesystem. Fall back to a
// replacement move; the hash directory guarantees content equality.
try {
Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException retry) {
Files.deleteIfExists(staging);
throw retry;
}
private static Path extractToPrivateTempDir(String resource, String fileName) throws IOException {
Path tmpDir = Files.createTempDirectory(TMP_DIR_PREFIX);
tmpDir.toFile().deleteOnExit();
Path target = tmpDir.resolve(fileName);
target.toFile().deleteOnExit();
try (InputStream in = NativeLibraryLoader.class.getResourceAsStream(resource)) {
Files.copy(in, target);
}
return target;
}

private static MessageDigest sha256() {
try {
return MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 not available", e);
}
}

private static String toHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(Character.forDigit((b >> 4) & 0xf, 16));
sb.append(Character.forDigit(b & 0xf, 16));
}
return sb.toString();
}

private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
err.initCause(cause);
Expand Down
21 changes: 10 additions & 11 deletions core/src/main/java/org/apache/datafusion/Platform.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@
import java.util.Locale;

/**
* Identifies a target OS/arch pair and the path at which the bundled
* native library is published inside the JAR.
* Identifies a target OS/arch pair and the path at which the bundled native library is published
* inside the JAR.
*
* <p>Resource layout follows the convention from the
* {@code datafusion-java} packaging design:
* <p>Resource layout follows the convention from the {@code datafusion-java} packaging design:
*
* <pre>
* org/apache/datafusion/linux/amd64/libdatafusion_jni.so
* org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
* org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
* org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
* org/apache/datafusion/windows/amd64/datafusion_jni.dll
* </pre>
*
* <p>Package-private; consumed only by {@link NativeLibraryLoader}.
Expand Down Expand Up @@ -90,15 +90,14 @@ static Os detectOs(String osName) {
if (n.startsWith("windows")) {
return Os.WINDOWS;
}
throw new UnsupportedOperationException(
"Unsupported OS for datafusion_jni: " + osName);
throw new UnsupportedOperationException("Unsupported OS for datafusion_jni: " + osName);
}

/**
* Returns the architecture segment used in the resource path for {@code os}.
*
* <p>Linux uses {@code amd64} (Java's preferred name for x86_64), while
* macOS uses {@code x86_64}; both use {@code aarch64} for ARM64.
* <p>Linux and Windows use {@code amd64} (Java's preferred name for x86_64), while macOS uses
* {@code x86_64}; all use {@code aarch64} for ARM64.
*/
static String detectArch(Os os, String osArch) {
if (osArch == null) {
Expand All @@ -108,7 +107,7 @@ static String detectArch(Os os, String osArch) {
boolean isX64 = n.equals("amd64") || n.equals("x86_64") || n.equals("x64");
boolean isArm64 = n.equals("aarch64") || n.equals("arm64");
if (isX64) {
return os == Os.LINUX ? "amd64" : "x86_64";
return os == Os.DARWIN ? "x86_64" : "amd64";
}
if (isArm64) {
return "aarch64";
Expand All @@ -122,8 +121,8 @@ String libFileName() {
}

/**
* Absolute classpath resource path (with leading slash) of the bundled
* native library for this platform.
* Absolute classpath resource path (with leading slash) of the bundled native library for this
* platform.
*/
String resourcePath() {
return "/" + RESOURCE_PREFIX + "/" + os.dirName + "/" + arch + "/" + libFileName();
Expand Down
19 changes: 14 additions & 5 deletions core/src/test/java/org/apache/datafusion/PlatformTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,12 @@ void detectsWindowsFromOsName() {

@Test
void rejectsUnknownOs() {
assertThrows(UnsupportedOperationException.class,
() -> Platform.detectOs("Solaris"));
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs("Solaris"));
}

@Test
void rejectsNullOs() {
assertThrows(UnsupportedOperationException.class,
() -> Platform.detectOs(null));
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs(null));
}

@Test
Expand All @@ -70,6 +68,13 @@ void usesX86_64OnDarwinForX86_64Aliases() {
assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "x86_64"));
}

@Test
void usesAmd64OnWindowsForX86_64Aliases() {
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "amd64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x86_64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x64"));
}

@Test
void usesAarch64ForArm64Aliases() {
assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "aarch64"));
Expand All @@ -80,7 +85,8 @@ void usesAarch64ForArm64Aliases() {

@Test
void rejectsUnknownArch() {
assertThrows(UnsupportedOperationException.class,
assertThrows(
UnsupportedOperationException.class,
() -> Platform.detectArch(Platform.Os.LINUX, "ppc64le"));
}

Expand All @@ -107,5 +113,8 @@ void resourcePathMatchesSpec() {
assertEquals(
"/org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib",
Platform.of("Mac OS X", "aarch64").resourcePath());
assertEquals(
"/org/apache/datafusion/windows/amd64/datafusion_jni.dll",
Platform.of("Windows 11", "amd64").resourcePath());
}
}
Loading