From ae0257a3a79d3ce65d942a4ddde0b2c9c88ac38f Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Sun, 23 Apr 2023 17:24:20 +1000 Subject: [PATCH] Zig cross-compilation of native libraries This new approach provides a number of benefits: 1. The LmdbJava Native project can be archived. This project required coordination with LmdbJava module releases and added extra artifacts to Maven Central simply to wrap native libraries. The native project also required a great deal of Maven configuration and it was always tedious to support additional platforms. Using Zig eliminates the need to run QEMU emulators in builds to support unusual platforms etc. 2. Zig supports numerous cross-compilation targets straight out of the box. A full list is available via zig targets | jq -r '.libc[]' and this presently lists 64 on my machine. This is likely to accommodate most LmdbJava platform support requests we are likely to receive. 3. Platform naming conventions have now been standardised based on the Zig target name. Support for individual build chains or specific processors is now simplified and much more transparent. 4. The GitHub Action has been amended to perform cross-compilation under Linux and upload the resulting native artifacts for later build steps. The later build steps run the Verifier on a native VM where available (eg Windows, Mac OS) and this therefore tests the cross-compiled libraries. 5. Target name resolution logic has been refactored and externalised in its own class with corresponding unit tests to ensure corner cases are duly considered and any bugs more easily reproduced and permanently rectified. This change is backwards compatible with users who used (and may continue to use) the lmdbjava.native.lib system property. --- .github/workflows/maven.yml | 27 +++ .gitignore | 1 + README.md | 20 ++- cross-compile.sh | 27 +++ pom.xml | 45 ----- src/main/java/org/lmdbjava/Library.java | 52 +----- src/main/java/org/lmdbjava/TargetName.java | 156 ++++++++++++++++++ src/main/resources/org/lmdbjava/.gitignore | 2 + .../java/org/lmdbjava/TargetNameTest.java | 74 +++++++++ 9 files changed, 310 insertions(+), 94 deletions(-) create mode 100755 cross-compile.sh create mode 100644 src/main/java/org/lmdbjava/TargetName.java create mode 100644 src/main/resources/org/lmdbjava/.gitignore create mode 100644 src/test/java/org/lmdbjava/TargetNameTest.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 85d4385f..4c3ac069 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -22,14 +22,29 @@ jobs: java-version: 17 cache: maven + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Build with Maven run: mvn -B verify + - name: Store built native libraries for later jobs + uses: actions/upload-artifact@v3 + with: + name: native-libraries + path: | + src/main/resources/org/lmdbjava/*.so + src/main/resources/org/lmdbjava/*.dll + - name: Upload code coverage to Codecov uses: codecov/codecov-action@v3 compatibility-checks: name: Java ${{ matrix.java }} on ${{ matrix.os }} Compatibility + needs: [build] runs-on: ${{ matrix.os }} strategy: @@ -48,6 +63,12 @@ jobs: java-version: ${{ matrix.java }} cache: maven + - name: Fetch built native libraries + uses: actions/download-artifact@v3 + with: + name: native-libraries + path: src/main/resources/org/lmdbjava + - name: Execute verifier run: mvn -B test -Dtest=VerifierTest -DverificationSeconds=10 @@ -81,6 +102,12 @@ jobs: gpg-private-key: ${{ secrets.gpg_private_key }} gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Publish Maven package run: mvn -B -Possrh-deploy deploy -DskipTests env: diff --git a/.gitignore b/.gitignore index fe6430cb..0c771329 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ dependency-reduced-pom.xml gpg-sign.json mvn-sync.json secrets.tar +lmdb diff --git a/README.md b/README.md index 450f814f..16ff78ae 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ * Modern, idiomatic Java API (including iterators, key ranges, enums, exceptions etc) * Nothing to install (the JAR embeds the latest LMDB libraries for Linux, OS X and Windows) * Buffer agnostic (Java `ByteBuffer`, Agrona `DirectBuffer`, Netty `ByteBuf`, your own buffer) -* 100% stock-standard, officially-released, widely-tested LMDB C code ([no extra](https://github.com/lmdbjava/native) C/JNI code) +* 100% stock-standard, officially-released, widely-tested LMDB C code (no extra C/JNI code) * Low latency design (allocation-free; buffer pools; optional checks can be easily disabled in production etc) * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data) * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues)) @@ -55,6 +55,24 @@ We're happy to help you use LmdbJava. Simply [open a GitHub issue](https://github.com/lmdbjava/lmdbjava/issues) if you have any questions. +### Building + +This project uses [Zig](https://ziglang.org/) to cross-compile the LMDB native +library for all supported architectures. To locally build LmdbJava you must +firstly install a recent version of Zig and then execute the project's +[cross-compile.sh](https://github.com/lmdbjava/lmdbjava/tree/master/cross-compile.sh) +script. This only needs to be repeated when the `cross-compile.sh` script is +updated (eg following a new official release of the upstream LMDB library). + +If you do not wish to install Zig and/or use an operating system which cannot +easily execute the `cross-compile.sh` script, you can download the compiled +LMDB native library for your platform from a location of your choice and set the +`lmdbjava.native.lib` system property to the resulting file system system +location. Possible sources of a compiled LMDB native library include operating +system package managers, running `cross-compile.sh` on a supported system, or +copying it from the `org/lmdbjava` directory of any recent, officially released +LmdbJava JAR. + ### Contributing Contributions are welcome! Please see the [Contributing Guidelines](CONTRIBUTING.md). diff --git a/cross-compile.sh b/cross-compile.sh new file mode 100755 index 00000000..1ebf92f6 --- /dev/null +++ b/cross-compile.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o errexit + +rm -rf lmdb +git clone --depth 1 --branch LMDB_0.9.29 https://github.com/LMDB/lmdb.git +pushd lmdb/libraries/liblmdb +trap popd SIGINT + +# zig targets | jq -r '.libc[]' +for target in aarch64-linux-gnu \ + aarch64-macos-none \ + x86_64-linux-gnu \ + x86_64-macos-none \ + x86_64-windows-gnu +do + echo "##### Building $target ####" + make -e clean liblmdb.so CC="zig cc -target $target" AR="zig ar" + if [[ "$target" == *-windows-* ]]; then + extension="dll" + else + extension="so" + fi + cp -v liblmdb.so ../../../src/main/resources/org/lmdbjava/$target.$extension +done + +ls -l ../../../src/main/resources/org/lmdbjava diff --git a/pom.xml b/pom.xml index 09b36edb..4c8f03ba 100644 --- a/pom.xml +++ b/pom.xml @@ -70,24 +70,6 @@ org.hamcrest hamcrest - - org.lmdbjava - lmdbjava-native-linux-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-osx-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-windows-x86_64 - 0.9.29-1 - true - @@ -103,11 +85,6 @@ org.apache.maven.plugins maven-dependency-plugin - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - com.github.jnr:jffi @@ -132,28 +109,6 @@ org.apache.maven.plugins maven-pmd-plugin - - org.apache.maven.plugins - maven-shade-plugin - - - lmdbjava-shade - - shade - - package - - - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - - - - - - org.apache.maven.plugins maven-surefire-plugin diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index e48def73..22308600 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -21,11 +21,8 @@ package org.lmdbjava; import static java.io.File.createTempFile; -import static java.lang.Boolean.getBoolean; import static java.lang.System.getProperty; import static java.lang.Thread.currentThread; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; import static jnr.ffi.LibraryLoader.create; import static jnr.ffi.Runtime.getRuntime; @@ -55,35 +52,15 @@ */ final class Library { - /** - * Java system property name that can be set to disable automatic extraction - * of the LMDB system library from the LmdbJava JAR. This may be desirable if - * an operating system-provided LMDB system library is preferred (eg operating - * system package management, vendor support, special compiler flags, security - * auditing, profile guided optimization builds, faster startup time by - * avoiding the library copy etc). - */ - public static final String DISABLE_EXTRACT_PROP = "lmdbjava.disable.extract"; /** * Java system property name that can be set to the path of an existing * directory into which the LMDB system library will be extracted from the * LmdbJava JAR. If unspecified the LMDB system library is extracted to the * java.io.tmpdir. Ignored if the LMDB system library is not * being extracted from the LmdbJava JAR (as would be the case if other - * system properties defined in Library have been set). + * system properties defined in TargetName have been set). */ public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir"; - /** - * Java system property name that can be set to provide a custom path to a - * external LMDB system library. If set, the system property - * DISABLE_EXTRACT_PROP will be overridden. - */ - public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; - /** - * Indicates whether automatic extraction of the LMDB system library is - * permitted. - */ - public static final boolean SHOULD_EXTRACT = !getBoolean(DISABLE_EXTRACT_PROP); /** * Indicates the directory where the LMDB system library will be extracted. */ @@ -91,35 +68,14 @@ final class Library { getProperty("java.io.tmpdir")); static final Lmdb LIB; static final jnr.ffi.Runtime RUNTIME; - /** - * Indicates whether external LMDB system library is provided. - */ - static final boolean SHOULD_USE_LIB = nonNull( - getProperty(LMDB_NATIVE_LIB_PROP)); - private static final String LIB_NAME = "lmdb"; static { final String libToLoad; - final String arch = getProperty("os.arch"); - final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch) - || "x86_64".equals(arch); - - final String os = getProperty("os.name"); - final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux"); - final boolean osx = os.startsWith("Mac OS X"); - final boolean windows = os.startsWith("Windows"); - - if (SHOULD_USE_LIB) { - libToLoad = getProperty(LMDB_NATIVE_LIB_PROP); - } else if (SHOULD_EXTRACT && arch64 && linux) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so"); - } else if (SHOULD_EXTRACT && arch64 && osx) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib"); - } else if (SHOULD_EXTRACT && arch64 && windows) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll"); + if (TargetName.IS_EXTERNAL) { + libToLoad = TargetName.RESOLVED_FILENAME; } else { - libToLoad = LIB_NAME; + libToLoad = extract(TargetName.RESOLVED_FILENAME); } LIB = create(Lmdb.class).load(libToLoad); diff --git a/src/main/java/org/lmdbjava/TargetName.java b/src/main/java/org/lmdbjava/TargetName.java new file mode 100644 index 00000000..3f8692a6 --- /dev/null +++ b/src/main/java/org/lmdbjava/TargetName.java @@ -0,0 +1,156 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static java.lang.System.getProperty; +import static java.util.Locale.ENGLISH; + +/** + * Determines the name of the target LMDB native library. + * + *

+ * Users will typically use an LMDB native library that is embedded within the + * LmdbJava JAR. Embedded libraries are built by a Zig cross-compilation step as + * part of the release process. The naming convention reflects the Zig target + * name plus a common filename extension. This simplifies support for future Zig + * targets (eg with different toolchains etc). + * + *

+ * Users can set two system properties to override the automatic resolution of + * an embedded library. Setting {@link #LMDB_NATIVE_LIB_PROP} will force use of + * that external LMDB library. Setting {@link #LMDB_EMBEDDED_LIB_PROP} will + * force use of that embedded LMDB library. If both are set, the former property + * will take precedence. Most users do not need to set either property. + */ +public final class TargetName { + + /** + * True if the resolved native filename is an external file (conversely false + * indicates the file should be considered a classpath resource). + */ + public static final boolean IS_EXTERNAL; + + /** + * Java system property name that can be set to override the embedded library + * that will be used. This is likely to required if automatic resolution fails + * but the user still prefers to use an LmdbJava-bundled library. + */ + public static final String LMDB_EMBEDDED_LIB_PROP = "lmdbjava.embedded.lib"; + /** + * Java system property name that can be set to provide a custom path to an + * external LMDB system library. This path must include the classpath prefix + * (usually org/lmdbjava). + */ + public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; + /** + * Resolved target native filename or fully-qualified classpath location. + */ + public static final String RESOLVED_FILENAME; + private static final String ARCH = getProperty("os.arch"); + private static final String EMBED = getProperty(LMDB_EMBEDDED_LIB_PROP); + private static final String EXTERNAL = getProperty(LMDB_NATIVE_LIB_PROP); + private static final String OS = getProperty("os.name"); + + static { + IS_EXTERNAL = isExternal(EXTERNAL); + RESOLVED_FILENAME = resolveFilename(EXTERNAL, EMBED, ARCH, OS); + } + + private TargetName() { + } + + public static String resolveExtension(final String os) { + return check(os, "Windows") ? "dll" : "so"; + } + + static boolean isExternal(final String external) { + return external != null && !external.isEmpty(); + } + + static String resolveFilename(final String external, final String embed, + final String arch, final String os) { + if (external != null && !external.isEmpty()) { + return external; + } + + if (embed != null && !embed.isEmpty()) { + return embed; + } + + return "org/lmdbjava/" + resolveArch(arch) + "-" + resolveOs(os) + "-" + + resolveToolchain(os) + "." + resolveExtension(os); + } + + /** + * Case insensitively checks whether the passed string starts with any of the + * candidate strings. + * + * @param string the string being checked + * @param candidates one or more candidate strings + * @return true if the string starts with any of the candidates + */ + private static boolean check(final String string, + final String... candidates) { + if (string == null) { + return false; + } + + final String strLower = string.toLowerCase(ENGLISH); + for (final String c : candidates) { + if (strLower.startsWith(c.toLowerCase(ENGLISH))) { + return true; + } + } + return false; + } + + private static String err(final String reason) { + return reason + " (please set system property " + LMDB_NATIVE_LIB_PROP + + " to the path of an external LMDB native library or property " + + LMDB_EMBEDDED_LIB_PROP + " to the name of an LmdbJava embedded" + + " library; os.arch='" + ARCH + "' os.name='" + OS + "')"; + } + + private static String resolveArch(final String arch) { + if (check(arch, "aarch64")) { + return "aarch64"; + } else if (check(arch, "x86_64", "amd64")) { + return "x86_64"; + } + throw new UnsupportedOperationException(err("Unsupported os.arch")); + } + + private static String resolveOs(final String os) { + if (check(os, "Linux")) { + return "linux"; + } else if (check(os, "Mac OS")) { + return "macos"; + } else if (check(os, "Windows")) { + return "windows"; + } + throw new UnsupportedOperationException(err("Unsupported os.name")); + } + + private static String resolveToolchain(final String os) { + return check(os, "Mac OS") ? "none" : "gnu"; + } + +} diff --git a/src/main/resources/org/lmdbjava/.gitignore b/src/main/resources/org/lmdbjava/.gitignore new file mode 100644 index 00000000..661f98b3 --- /dev/null +++ b/src/main/resources/org/lmdbjava/.gitignore @@ -0,0 +1,2 @@ +*.so +*.dll diff --git a/src/test/java/org/lmdbjava/TargetNameTest.java b/src/test/java/org/lmdbjava/TargetNameTest.java new file mode 100644 index 00000000..eec38233 --- /dev/null +++ b/src/test/java/org/lmdbjava/TargetNameTest.java @@ -0,0 +1,74 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.lmdbjava.TargetName.isExternal; +import static org.lmdbjava.TargetName.resolveFilename; +import static org.lmdbjava.TestUtils.invokePrivateConstructor; + +import org.junit.Test; + +/** + * Test {@link TargetName}. + */ +public final class TargetNameTest { + + private static final String NONE = ""; + + @Test + public void coverPrivateConstructors() { + invokePrivateConstructor(TargetName.class); + } + + @Test + public void customEmbedded() { + assertThat(resolveFilename(NONE, "x/y.so", NONE, NONE), is("x/y.so")); + assertThat(isExternal(NONE), is(false)); + } + + @Test + public void embeddedNameResolution() { + embed("aarch64-linux-gnu.so", "aarch64", "Linux"); + embed("aarch64-macos-none.so", "aarch64", "Mac OS"); + embed("x86_64-linux-gnu.so", "x86_64", "Linux"); + embed("x86_64-macos-none.so", "x86_64", "Mac OS"); + embed("x86_64-windows-gnu.dll", "x86_64", "Windows"); + } + + @Test + public void externalLibrary() { + assertThat(resolveFilename("/l.so", NONE, NONE, NONE), is("/l.so")); + assertThat(TargetName.isExternal("/l.so"), is(true)); + } + + @Test + public void externalTakesPriority() { + assertThat(resolveFilename("/lm.so", "x/y.so", NONE, NONE), is("/lm.so")); + assertThat(isExternal("/lm.so"), is(true)); + } + + private void embed(final String lib, final String arch, final String os) { + assertThat(resolveFilename(NONE, NONE, arch, os), is("org/lmdbjava/" + lib)); + assertThat(isExternal(NONE), is(false)); + } +}