lmdbjava · benalexau · Apr 24, 2023 · Apr 23, 2023
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -22,14 +22,29 @@ jobs:
         java-version: 17
         cache: maven
 
+    - name: Install Zig
+      uses: goto-bus-stop/setup-zig@v2
+
+    - name: Cross compile using Zig
+      run: ./cross-compile.sh
+
     - name: Build with Maven
       run: mvn -B verify
 
+    - name: Store built native libraries for later jobs
+      uses: actions/upload-artifact@v3
+      with:
+        name: native-libraries
+        path: |
+          src/main/resources/org/lmdbjava/*.so
+          src/main/resources/org/lmdbjava/*.dll
+
     - name: Upload code coverage to Codecov
       uses: codecov/codecov-action@v3
 
   compatibility-checks:
     name: Java ${{ matrix.java }} on ${{ matrix.os }} Compatibility
+    needs: [build]
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -48,6 +63,12 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: maven
 
+      - name: Fetch built native libraries
+        uses: actions/download-artifact@v3
+        with:
+          name: native-libraries
+          path: src/main/resources/org/lmdbjava
+
       - name: Execute verifier
         run: mvn -B test -Dtest=VerifierTest -DverificationSeconds=10
 
@@ -81,6 +102,12 @@ jobs:
         gpg-private-key: ${{ secrets.gpg_private_key }}
         gpg-passphrase: MAVEN_GPG_PASSPHRASE
 
+    - name: Install Zig
+      uses: goto-bus-stop/setup-zig@v2
+
+    - name: Cross compile using Zig
+      run: ./cross-compile.sh
+
     - name: Publish Maven package
       run: mvn -B -Possrh-deploy deploy -DskipTests
       env:

diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,4 @@ dependency-reduced-pom.xml
 gpg-sign.json
 mvn-sync.json
 secrets.tar
+lmdb
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@
 * Modern, idiomatic Java API (including iterators, key ranges, enums, exceptions etc)
 * Nothing to install (the JAR embeds the latest LMDB libraries for Linux, OS X and Windows)
 * Buffer agnostic (Java `ByteBuffer`, Agrona `DirectBuffer`, Netty `ByteBuf`, your own buffer)
-* 100% stock-standard, officially-released, widely-tested LMDB C code ([no extra](https://github.com/lmdbjava/native) C/JNI code)
+* 100% stock-standard, officially-released, widely-tested LMDB C code (no extra C/JNI code)
 * Low latency design (allocation-free; buffer pools; optional checks can be easily disabled in production etc)
 * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data)
 * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues))
@@ -55,6 +55,24 @@ We're happy to help you use LmdbJava. Simply
 [open a GitHub issue](https://github.com/lmdbjava/lmdbjava/issues) if you have
 any questions.
 
+### Building
+
+This project uses [Zig](https://ziglang.org/) to cross-compile the LMDB native
+library for all supported architectures. To locally build LmdbJava you must
+firstly install a recent version of Zig and then execute the project's
+[cross-compile.sh](https://github.com/lmdbjava/lmdbjava/tree/master/cross-compile.sh)
+script. This only needs to be repeated when the `cross-compile.sh` script is
+updated (eg following a new official release of the upstream LMDB library).
+
+If you do not wish to install Zig and/or use an operating system which cannot
+easily execute the `cross-compile.sh` script, you can download the compiled
+LMDB native library for your platform from a location of your choice and set the
+`lmdbjava.native.lib` system property to the resulting file system system
+location. Possible sources of a compiled LMDB native library include operating
+system package managers, running `cross-compile.sh` on a supported system, or
+copying it from the `org/lmdbjava` directory of any recent, officially released
+LmdbJava JAR.
+
 ### Contributing
 
 Contributions are welcome! Please see the [Contributing Guidelines](CONTRIBUTING.md).

diff --git a/cross-compile.sh b/cross-compile.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -o errexit
+
+rm -rf lmdb
+git clone --depth 1 --branch LMDB_0.9.29 https://github.com/LMDB/lmdb.git
+pushd lmdb/libraries/liblmdb
+trap popd SIGINT
+
+# zig targets | jq -r '.libc[]'
+for target in aarch64-linux-gnu \
+              aarch64-macos-none \
+              x86_64-linux-gnu \
+              x86_64-macos-none \
+              x86_64-windows-gnu
+do
+  echo "##### Building $target ####"
+  make -e clean liblmdb.so CC="zig cc -target $target" AR="zig ar"
+  if [[ "$target" == *-windows-* ]]; then
+    extension="dll"
+  else
+    extension="so"
+  fi
+  cp -v liblmdb.so ../../../src/main/resources/org/lmdbjava/$target.$extension
+done
+
+ls -l ../../../src/main/resources/org/lmdbjava
diff --git a/pom.xml b/pom.xml
@@ -70,24 +70,6 @@
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.lmdbjava</groupId>
-      <artifactId>lmdbjava-native-linux-x86_64</artifactId>
-      <version>0.9.29-1</version>
-      <optional>true</optional>
-    </dependency>
-    <dependency>
-      <groupId>org.lmdbjava</groupId>
-      <artifactId>lmdbjava-native-osx-x86_64</artifactId>
-      <version>0.9.29-1</version>
-      <optional>true</optional>
-    </dependency>
-    <dependency>
-      <groupId>org.lmdbjava</groupId>
-      <artifactId>lmdbjava-native-windows-x86_64</artifactId>
-      <version>0.9.29-1</version>
-      <optional>true</optional>
-    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -103,11 +85,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
         <configuration>
-          <usedDependencies>
-            <usedDependency>org.lmdbjava:lmdbjava-native-linux-x86_64</usedDependency>
-            <usedDependency>org.lmdbjava:lmdbjava-native-windows-x86_64</usedDependency>
-            <usedDependency>org.lmdbjava:lmdbjava-native-osx-x86_64</usedDependency>
-          </usedDependencies>
           <ignoredDependencies>
             <ignoredDependency>com.github.jnr:jffi</ignoredDependency>
           </ignoredDependencies>
@@ -132,28 +109,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-pmd-plugin</artifactId>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>lmdbjava-shade</id>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <phase>package</phase>
-            <configuration>
-              <artifactSet>
-                <includes>
-                  <include>org.lmdbjava:lmdbjava-native-linux-x86_64</include>
-                  <include>org.lmdbjava:lmdbjava-native-windows-x86_64</include>
-                  <include>org.lmdbjava:lmdbjava-native-osx-x86_64</include>
-                </includes>
-              </artifactSet>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>

diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java
@@ -21,11 +21,8 @@
 package org.lmdbjava;
 
 import static java.io.File.createTempFile;
-import static java.lang.Boolean.getBoolean;
 import static java.lang.System.getProperty;
 import static java.lang.Thread.currentThread;
-import static java.util.Locale.ENGLISH;
-import static java.util.Objects.nonNull;
 import static java.util.Objects.requireNonNull;
 import static jnr.ffi.LibraryLoader.create;
 import static jnr.ffi.Runtime.getRuntime;
@@ -55,71 +52,30 @@
  */
 final class Library {
 
-  /**
-   * Java system property name that can be set to disable automatic extraction
-   * of the LMDB system library from the LmdbJava JAR. This may be desirable if
-   * an operating system-provided LMDB system library is preferred (eg operating
-   * system package management, vendor support, special compiler flags, security
-   * auditing, profile guided optimization builds, faster startup time by
-   * avoiding the library copy etc).
-   */
-  public static final String DISABLE_EXTRACT_PROP = "lmdbjava.disable.extract";
   /**
    * Java system property name that can be set to the path of an existing
    * directory into which the LMDB system library will be extracted from the
    * LmdbJava JAR. If unspecified the LMDB system library is extracted to the
    * <code>java.io.tmpdir</code>. Ignored if the LMDB system library is not
    * being extracted from the LmdbJava JAR (as would be the case if other
-   * system properties defined in <code>Library</code> have been set).
+   * system properties defined in <code>TargetName</code> have been set).
    */
   public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir";
-  /**
-   * Java system property name that can be set to provide a custom path to a
-   * external LMDB system library. If set, the system property
-   * DISABLE_EXTRACT_PROP will be overridden.
-   */
-  public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib";
-  /**
-   * Indicates whether automatic extraction of the LMDB system library is
-   * permitted.
-   */
-  public static final boolean SHOULD_EXTRACT = !getBoolean(DISABLE_EXTRACT_PROP);
   /**
    * Indicates the directory where the LMDB system library will be extracted.
    */
   static final String EXTRACT_DIR = getProperty(LMDB_EXTRACT_DIR_PROP,
                                                 getProperty("java.io.tmpdir"));
   static final Lmdb LIB;
   static final jnr.ffi.Runtime RUNTIME;
-  /**
-   * Indicates whether external LMDB system library is provided.
-   */
-  static final boolean SHOULD_USE_LIB = nonNull(
-      getProperty(LMDB_NATIVE_LIB_PROP));
-  private static final String LIB_NAME = "lmdb";
 
   static {
     final String libToLoad;
 
-    final String arch = getProperty("os.arch");
-    final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch)
-                               || "x86_64".equals(arch);
-
-    final String os = getProperty("os.name");
-    final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux");
-    final boolean osx = os.startsWith("Mac OS X");
-    final boolean windows = os.startsWith("Windows");
-
-    if (SHOULD_USE_LIB) {
-      libToLoad = getProperty(LMDB_NATIVE_LIB_PROP);
-    } else if (SHOULD_EXTRACT && arch64 && linux) {
-      libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so");
-    } else if (SHOULD_EXTRACT && arch64 && osx) {
-      libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib");
-    } else if (SHOULD_EXTRACT && arch64 && windows) {
-      libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll");
+    if (TargetName.IS_EXTERNAL) {
+      libToLoad = TargetName.RESOLVED_FILENAME;
     } else {
-      libToLoad = LIB_NAME;
+      libToLoad = extract(TargetName.RESOLVED_FILENAME);
     }
 
     LIB = create(Lmdb.class).load(libToLoad);