diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 85d4385..4c3ac06 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -22,14 +22,29 @@ jobs: java-version: 17 cache: maven + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Build with Maven run: mvn -B verify + - name: Store built native libraries for later jobs + uses: actions/upload-artifact@v3 + with: + name: native-libraries + path: | + src/main/resources/org/lmdbjava/*.so + src/main/resources/org/lmdbjava/*.dll + - name: Upload code coverage to Codecov uses: codecov/codecov-action@v3 compatibility-checks: name: Java ${{ matrix.java }} on ${{ matrix.os }} Compatibility + needs: [build] runs-on: ${{ matrix.os }} strategy: @@ -48,6 +63,12 @@ jobs: java-version: ${{ matrix.java }} cache: maven + - name: Fetch built native libraries + uses: actions/download-artifact@v3 + with: + name: native-libraries + path: src/main/resources/org/lmdbjava + - name: Execute verifier run: mvn -B test -Dtest=VerifierTest -DverificationSeconds=10 @@ -81,6 +102,12 @@ jobs: gpg-private-key: ${{ secrets.gpg_private_key }} gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Publish Maven package run: mvn -B -Possrh-deploy deploy -DskipTests env: diff --git a/.gitignore b/.gitignore index fe6430c..0c77132 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ dependency-reduced-pom.xml gpg-sign.json mvn-sync.json secrets.tar +lmdb diff --git a/README.md b/README.md index 450f814..16ff78a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ * Modern, idiomatic Java API (including iterators, key ranges, enums, exceptions etc) * Nothing to install (the JAR embeds the latest LMDB libraries for Linux, OS X and Windows) * Buffer agnostic (Java `ByteBuffer`, Agrona `DirectBuffer`, Netty `ByteBuf`, your own buffer) -* 100% stock-standard, officially-released, widely-tested LMDB C code ([no extra](https://github.com/lmdbjava/native) C/JNI code) +* 100% stock-standard, officially-released, widely-tested LMDB C code (no extra C/JNI code) * Low latency design (allocation-free; buffer pools; optional checks can be easily disabled in production etc) * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data) * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues)) @@ -55,6 +55,24 @@ We're happy to help you use LmdbJava. Simply [open a GitHub issue](https://github.com/lmdbjava/lmdbjava/issues) if you have any questions. +### Building + +This project uses [Zig](https://ziglang.org/) to cross-compile the LMDB native +library for all supported architectures. To locally build LmdbJava you must +firstly install a recent version of Zig and then execute the project's +[cross-compile.sh](https://github.com/lmdbjava/lmdbjava/tree/master/cross-compile.sh) +script. This only needs to be repeated when the `cross-compile.sh` script is +updated (eg following a new official release of the upstream LMDB library). + +If you do not wish to install Zig and/or use an operating system which cannot +easily execute the `cross-compile.sh` script, you can download the compiled +LMDB native library for your platform from a location of your choice and set the +`lmdbjava.native.lib` system property to the resulting file system system +location. Possible sources of a compiled LMDB native library include operating +system package managers, running `cross-compile.sh` on a supported system, or +copying it from the `org/lmdbjava` directory of any recent, officially released +LmdbJava JAR. + ### Contributing Contributions are welcome! Please see the [Contributing Guidelines](CONTRIBUTING.md). diff --git a/cross-compile.sh b/cross-compile.sh new file mode 100755 index 0000000..1ebf92f --- /dev/null +++ b/cross-compile.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o errexit + +rm -rf lmdb +git clone --depth 1 --branch LMDB_0.9.29 https://github.com/LMDB/lmdb.git +pushd lmdb/libraries/liblmdb +trap popd SIGINT + +# zig targets | jq -r '.libc[]' +for target in aarch64-linux-gnu \ + aarch64-macos-none \ + x86_64-linux-gnu \ + x86_64-macos-none \ + x86_64-windows-gnu +do + echo "##### Building $target ####" + make -e clean liblmdb.so CC="zig cc -target $target" AR="zig ar" + if [[ "$target" == *-windows-* ]]; then + extension="dll" + else + extension="so" + fi + cp -v liblmdb.so ../../../src/main/resources/org/lmdbjava/$target.$extension +done + +ls -l ../../../src/main/resources/org/lmdbjava diff --git a/pom.xml b/pom.xml index 09b36ed..4c8f03b 100644 --- a/pom.xml +++ b/pom.xml @@ -70,24 +70,6 @@ org.hamcrest hamcrest - - org.lmdbjava - lmdbjava-native-linux-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-osx-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-windows-x86_64 - 0.9.29-1 - true - @@ -103,11 +85,6 @@ org.apache.maven.plugins maven-dependency-plugin - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - com.github.jnr:jffi @@ -132,28 +109,6 @@ org.apache.maven.plugins maven-pmd-plugin - - org.apache.maven.plugins - maven-shade-plugin - - - lmdbjava-shade - - shade - - package - - - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - - - - - - org.apache.maven.plugins maven-surefire-plugin diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index e48def7..2230860 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -21,11 +21,8 @@ package org.lmdbjava; import static java.io.File.createTempFile; -import static java.lang.Boolean.getBoolean; import static java.lang.System.getProperty; import static java.lang.Thread.currentThread; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; import static jnr.ffi.LibraryLoader.create; import static jnr.ffi.Runtime.getRuntime; @@ -55,35 +52,15 @@ */ final class Library { - /** - * Java system property name that can be set to disable automatic extraction - * of the LMDB system library from the LmdbJava JAR. This may be desirable if - * an operating system-provided LMDB system library is preferred (eg operating - * system package management, vendor support, special compiler flags, security - * auditing, profile guided optimization builds, faster startup time by - * avoiding the library copy etc). - */ - public static final String DISABLE_EXTRACT_PROP = "lmdbjava.disable.extract"; /** * Java system property name that can be set to the path of an existing * directory into which the LMDB system library will be extracted from the * LmdbJava JAR. If unspecified the LMDB system library is extracted to the * java.io.tmpdir. Ignored if the LMDB system library is not * being extracted from the LmdbJava JAR (as would be the case if other - * system properties defined in Library have been set). + * system properties defined in TargetName have been set). */ public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir"; - /** - * Java system property name that can be set to provide a custom path to a - * external LMDB system library. If set, the system property - * DISABLE_EXTRACT_PROP will be overridden. - */ - public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; - /** - * Indicates whether automatic extraction of the LMDB system library is - * permitted. - */ - public static final boolean SHOULD_EXTRACT = !getBoolean(DISABLE_EXTRACT_PROP); /** * Indicates the directory where the LMDB system library will be extracted. */ @@ -91,35 +68,14 @@ final class Library { getProperty("java.io.tmpdir")); static final Lmdb LIB; static final jnr.ffi.Runtime RUNTIME; - /** - * Indicates whether external LMDB system library is provided. - */ - static final boolean SHOULD_USE_LIB = nonNull( - getProperty(LMDB_NATIVE_LIB_PROP)); - private static final String LIB_NAME = "lmdb"; static { final String libToLoad; - final String arch = getProperty("os.arch"); - final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch) - || "x86_64".equals(arch); - - final String os = getProperty("os.name"); - final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux"); - final boolean osx = os.startsWith("Mac OS X"); - final boolean windows = os.startsWith("Windows"); - - if (SHOULD_USE_LIB) { - libToLoad = getProperty(LMDB_NATIVE_LIB_PROP); - } else if (SHOULD_EXTRACT && arch64 && linux) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so"); - } else if (SHOULD_EXTRACT && arch64 && osx) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib"); - } else if (SHOULD_EXTRACT && arch64 && windows) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll"); + if (TargetName.IS_EXTERNAL) { + libToLoad = TargetName.RESOLVED_FILENAME; } else { - libToLoad = LIB_NAME; + libToLoad = extract(TargetName.RESOLVED_FILENAME); } LIB = create(Lmdb.class).load(libToLoad); diff --git a/src/main/java/org/lmdbjava/TargetName.java b/src/main/java/org/lmdbjava/TargetName.java new file mode 100644 index 0000000..3f8692a --- /dev/null +++ b/src/main/java/org/lmdbjava/TargetName.java @@ -0,0 +1,156 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static java.lang.System.getProperty; +import static java.util.Locale.ENGLISH; + +/** + * Determines the name of the target LMDB native library. + * + *

+ * Users will typically use an LMDB native library that is embedded within the + * LmdbJava JAR. Embedded libraries are built by a Zig cross-compilation step as + * part of the release process. The naming convention reflects the Zig target + * name plus a common filename extension. This simplifies support for future Zig + * targets (eg with different toolchains etc). + * + *

+ * Users can set two system properties to override the automatic resolution of + * an embedded library. Setting {@link #LMDB_NATIVE_LIB_PROP} will force use of + * that external LMDB library. Setting {@link #LMDB_EMBEDDED_LIB_PROP} will + * force use of that embedded LMDB library. If both are set, the former property + * will take precedence. Most users do not need to set either property. + */ +public final class TargetName { + + /** + * True if the resolved native filename is an external file (conversely false + * indicates the file should be considered a classpath resource). + */ + public static final boolean IS_EXTERNAL; + + /** + * Java system property name that can be set to override the embedded library + * that will be used. This is likely to required if automatic resolution fails + * but the user still prefers to use an LmdbJava-bundled library. + */ + public static final String LMDB_EMBEDDED_LIB_PROP = "lmdbjava.embedded.lib"; + /** + * Java system property name that can be set to provide a custom path to an + * external LMDB system library. This path must include the classpath prefix + * (usually org/lmdbjava). + */ + public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; + /** + * Resolved target native filename or fully-qualified classpath location. + */ + public static final String RESOLVED_FILENAME; + private static final String ARCH = getProperty("os.arch"); + private static final String EMBED = getProperty(LMDB_EMBEDDED_LIB_PROP); + private static final String EXTERNAL = getProperty(LMDB_NATIVE_LIB_PROP); + private static final String OS = getProperty("os.name"); + + static { + IS_EXTERNAL = isExternal(EXTERNAL); + RESOLVED_FILENAME = resolveFilename(EXTERNAL, EMBED, ARCH, OS); + } + + private TargetName() { + } + + public static String resolveExtension(final String os) { + return check(os, "Windows") ? "dll" : "so"; + } + + static boolean isExternal(final String external) { + return external != null && !external.isEmpty(); + } + + static String resolveFilename(final String external, final String embed, + final String arch, final String os) { + if (external != null && !external.isEmpty()) { + return external; + } + + if (embed != null && !embed.isEmpty()) { + return embed; + } + + return "org/lmdbjava/" + resolveArch(arch) + "-" + resolveOs(os) + "-" + + resolveToolchain(os) + "." + resolveExtension(os); + } + + /** + * Case insensitively checks whether the passed string starts with any of the + * candidate strings. + * + * @param string the string being checked + * @param candidates one or more candidate strings + * @return true if the string starts with any of the candidates + */ + private static boolean check(final String string, + final String... candidates) { + if (string == null) { + return false; + } + + final String strLower = string.toLowerCase(ENGLISH); + for (final String c : candidates) { + if (strLower.startsWith(c.toLowerCase(ENGLISH))) { + return true; + } + } + return false; + } + + private static String err(final String reason) { + return reason + " (please set system property " + LMDB_NATIVE_LIB_PROP + + " to the path of an external LMDB native library or property " + + LMDB_EMBEDDED_LIB_PROP + " to the name of an LmdbJava embedded" + + " library; os.arch='" + ARCH + "' os.name='" + OS + "')"; + } + + private static String resolveArch(final String arch) { + if (check(arch, "aarch64")) { + return "aarch64"; + } else if (check(arch, "x86_64", "amd64")) { + return "x86_64"; + } + throw new UnsupportedOperationException(err("Unsupported os.arch")); + } + + private static String resolveOs(final String os) { + if (check(os, "Linux")) { + return "linux"; + } else if (check(os, "Mac OS")) { + return "macos"; + } else if (check(os, "Windows")) { + return "windows"; + } + throw new UnsupportedOperationException(err("Unsupported os.name")); + } + + private static String resolveToolchain(final String os) { + return check(os, "Mac OS") ? "none" : "gnu"; + } + +} diff --git a/src/main/resources/org/lmdbjava/.gitignore b/src/main/resources/org/lmdbjava/.gitignore new file mode 100644 index 0000000..661f98b --- /dev/null +++ b/src/main/resources/org/lmdbjava/.gitignore @@ -0,0 +1,2 @@ +*.so +*.dll diff --git a/src/test/java/org/lmdbjava/TargetNameTest.java b/src/test/java/org/lmdbjava/TargetNameTest.java new file mode 100644 index 0000000..eec3823 --- /dev/null +++ b/src/test/java/org/lmdbjava/TargetNameTest.java @@ -0,0 +1,74 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.lmdbjava.TargetName.isExternal; +import static org.lmdbjava.TargetName.resolveFilename; +import static org.lmdbjava.TestUtils.invokePrivateConstructor; + +import org.junit.Test; + +/** + * Test {@link TargetName}. + */ +public final class TargetNameTest { + + private static final String NONE = ""; + + @Test + public void coverPrivateConstructors() { + invokePrivateConstructor(TargetName.class); + } + + @Test + public void customEmbedded() { + assertThat(resolveFilename(NONE, "x/y.so", NONE, NONE), is("x/y.so")); + assertThat(isExternal(NONE), is(false)); + } + + @Test + public void embeddedNameResolution() { + embed("aarch64-linux-gnu.so", "aarch64", "Linux"); + embed("aarch64-macos-none.so", "aarch64", "Mac OS"); + embed("x86_64-linux-gnu.so", "x86_64", "Linux"); + embed("x86_64-macos-none.so", "x86_64", "Mac OS"); + embed("x86_64-windows-gnu.dll", "x86_64", "Windows"); + } + + @Test + public void externalLibrary() { + assertThat(resolveFilename("/l.so", NONE, NONE, NONE), is("/l.so")); + assertThat(TargetName.isExternal("/l.so"), is(true)); + } + + @Test + public void externalTakesPriority() { + assertThat(resolveFilename("/lm.so", "x/y.so", NONE, NONE), is("/lm.so")); + assertThat(isExternal("/lm.so"), is(true)); + } + + private void embed(final String lib, final String arch, final String os) { + assertThat(resolveFilename(NONE, NONE, arch, os), is("org/lmdbjava/" + lib)); + assertThat(isExternal(NONE), is(false)); + } +}