diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index bc9d5f8..01b89bc 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -13,11 +13,19 @@ jobs: MAVEN_CENTRAL_PGP_KEY: ${{ secrets.MAVEN_CENTRAL_PGP_KEY }} steps: - - uses: actions/checkout@v1 - - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - uses: gradle/actions/wrapper-validation@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: '8.0.282' + java-version: '11' + distribution: 'temurin' + check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + env: + CI: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 13a366a..f16bf96 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,15 +7,25 @@ on: pull_request: branches: - master + - reactive-streams-branch + - '**' jobs: buildAndTest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - uses: gradle/actions/wrapper-validation@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: '8.0.282' + java-version: '11' + distribution: 'temurin' + check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace + env: + CI: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b61d755..a574a68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,11 +17,19 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.version }} steps: - - uses: actions/checkout@v1 - - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - uses: gradle/actions/wrapper-validation@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v4 with: - java-version: '8.0.282' + java-version: '11' + distribution: 'temurin' + check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + env: + CI: true diff --git a/.github/workflows/stale-pr-issue.yml b/.github/workflows/stale-pr-issue.yml new file mode 100644 index 0000000..d945402 --- /dev/null +++ b/.github/workflows/stale-pr-issue.yml @@ -0,0 +1,48 @@ +# Mark inactive issues and PRs as stale +# GitHub action based on https://github.com/actions/stale + +name: 'Close stale issues and PRs' +on: + schedule: + # Execute every day + - cron: '0 0 * * *' + +permissions: + actions: write + issues: write + pull-requests: write + +jobs: + close-pending: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + # GLOBAL ------------------------------------------------------------ + # Exempt any PRs or issues already added to a milestone + exempt-all-milestones: true + # Days until issues or pull requests are labelled as stale + days-before-stale: 60 + + # ISSUES ------------------------------------------------------------ + # Issues will be closed after 90 days of inactive (60 to mark as stale + 30 to close) + days-before-issue-close: 30 + stale-issue-message: > + Hello, this issue has been inactive for 60 days, so we're marking it as stale. + If you would like to continue this discussion, please comment within the next 30 days or we'll close the issue. + close-issue-message: > + Hello, as this issue has been inactive for 90 days, we're closing the issue. + If you would like to resume the discussion, please create a new issue. + exempt-issue-labels: keep-open + + # PULL REQUESTS ----------------------------------------------------- + # PRs will be closed after 90 days of inactive (60 to mark as stale + 30 to close) + days-before-pr-close: 30 + stale-pr-message: > + Hello, this pull request has been inactive for 60 days, so we're marking it as stale. + If you would like to continue working on this pull request, please make an update within the next 30 days, or we'll close the pull request. + close-pr-message: > + Hello, as this pull request has been inactive for 90 days, we're closing this pull request. + We always welcome contributions, and if you would like to continue, please open a new pull request. + exempt-pr-labels: keep-open + \ No newline at end of file diff --git a/README.md b/README.md index e48de1d..a53e766 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # java-dataloader [![Build](https://github.com/graphql-java/java-dataloader/actions/workflows/master.yml/badge.svg)](https://github.com/graphql-java/java-dataloader/actions/workflows/master.yml) -[![Latest Release](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/) +[![Latest Release](https://img.shields.io/maven-central/v/com.graphql-java/java-dataloader?versionPrefix=4.)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/graphql-java/) +[![Latest Snapshot](https://img.shields.io/maven-central/v/com.graphql-java/java-dataloader?label=maven-central%20snapshot&versionPrefix=0)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/graphql-java/) [![Apache licensed](https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000)](https://github.com/graphql-java/java-dataloader/blob/master/LICENSE) -This small and simple utility library is a pure Java 8 port of [Facebook DataLoader](https://github.com/facebook/dataloader). +This small and simple utility library is a pure Java 11 port of [Facebook DataLoader](https://github.com/facebook/dataloader). It can serve as integral part of your application's data layer to provide a consistent API over various back-ends and reduce message communication overhead through batching and caching. @@ -15,7 +16,7 @@ are resolved independently and, with a true graph of objects, you may be fetchin A naive implementation of graphql data fetchers can easily lead to the dreaded "n+1" fetch problem. Most of the code is ported directly from Facebook's reference implementation, with one IMPORTANT adaptation to make -it work for Java 8. ([more on this below](#manual-dispatching)). +it work for Java 11. ([more on this below](#manual-dispatching)). Before reading on, be sure to take a short dive into the [original documentation](https://github.com/facebook/dataloader/blob/master/README.md) provided by Lee Byron (@leebyron) @@ -63,11 +64,11 @@ Gradle users configure the `java-dataloader` dependency in `build.gradle`: ``` repositories { - jcenter() + mavenCentral() } dependencies { - compile 'com.graphql-java:java-dataloader: 3.1.0' + compile 'com.graphql-java:java-dataloader: 4.0.0' } ``` @@ -204,7 +205,7 @@ for the context object. ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -226,7 +227,7 @@ You can gain access to them as a map by key or as the original list of context o ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -286,6 +287,77 @@ For example, let's assume you want to load users from a database, you could prob // ... ``` +### Returning a stream of results from your batch publisher + +It may be that your batch loader function can use a [Reactive Streams](https://www.reactive-streams.org/) [Publisher](https://www.reactive-streams.org/reactive-streams-1.0.3-javadoc/org/reactivestreams/Publisher.html), where values are emitted as an asynchronous stream. + +For example, let's say you wanted to load many users from a service without forcing the service to load all +users into its memory (which may exert considerable pressure on it). + +A `org.dataloader.BatchPublisher` may be used to load this data: + +```java + BatchPublisher batchPublisher = new BatchPublisher() { + @Override + public void load(List userIds, Subscriber userSubscriber) { + Publisher userResults = userManager.streamUsersById(userIds); + userResults.subscribe(userSubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); + + // ... +``` + +Rather than waiting for all user values to be returned on one batch, this `DataLoader` will complete +the `CompletableFuture` returned by `Dataloader#load(Long)` as each value is +published. + +This pattern means that data loader values can (in theory) be satisfied more quickly than if we wait for +all results in the batch to be retrieved and hence the overall result may finish more quickly. + +If an exception is thrown, the remaining futures yet to be completed are completed +exceptionally. + +You *MUST* ensure that the values are streamed in the same order as the keys provided, +with the same cardinality (i.e. the number of values must match the number of keys). + +Failing to do so will result in incorrect data being returned from `DataLoader#load`. + +`BatchPublisher` is the reactive version of `BatchLoader`. + + +### Returning a mapped stream of results from your batch publisher + +Your publisher may not necessarily return values in the same order in which it processes keys and it +may not be able to find a value for each key presented. + +For example, let's say your batch publisher function loads user data which is spread across shards, +with some shards responding more quickly than others. + +In instances like these, `org.dataloader.MappedBatchPublisher` can be used. + +```java + MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { + @Override + public void load(Set userIds, Subscriber> userEntrySubscriber) { + Publisher> userEntries = userManager.streamUsersById(userIds); + userEntries.subscribe(userEntrySubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); + + // ... +``` + +Like the `BatchPublisher`, if an exception is thrown, the remaining futures yet to be completed are completed +exceptionally. + +Unlike the `BatchPublisher`, however, it is not necessary to return values in the same order as the provided keys, +or even the same number of values. + +`MappedBatchPublisher` is the reactive version of `MappedBatchLoader`. + ### Error object is not a thing in a type safe Java world In the reference JS implementation if the batch loader returns an `Error` object back from the `load()` promise is rejected @@ -361,7 +433,7 @@ However, you can create your own custom future cache and supply it to the data l ```java MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache).build(); DataLoaderFactory.newDataLoader(userBatchLoader, options); ``` @@ -395,7 +467,7 @@ The tests have an example based on [Caffeine](https://github.com/ben-manes/caffe In certain uncommon cases, a DataLoader which does not cache may be desirable. ```java - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false).build()); ``` Calling the above will ensure that every call to `.load()` will produce a new promise, and requested keys will not be saved in memory. @@ -461,7 +533,7 @@ Knowing what the behaviour of your data is important for you to understand how e You can configure the statistics collector used when you build the data loader ```java - DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()).build(); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader,options); ``` @@ -510,7 +582,58 @@ and there are also gains to this different mode of operation: However, with batch execution control comes responsibility! If you forget to make the call to `dispatch()` then the futures in the load request queue will never be batched, and thus _will never complete_! So be careful when crafting your loader designs. -## Scheduled Dispatching +## The BatchLoader Scheduler + +By default, when `dataLoader.dispatch()` is called, the `BatchLoader` / `MappedBatchLoader` function will be invoked +immediately. + +However, you can provide your own `BatchLoaderScheduler` that allows this call to be done some time into +the future. + +You will be passed a callback (`ScheduledBatchLoaderCall` / `ScheduledMapBatchLoaderCall`) and you are expected +to eventually call this callback method to make the batch loading happen. + +The following is a `BatchLoaderScheduler` that waits 10 milliseconds before invoking the batch loading functions. + +```java + new BatchLoaderScheduler() { + + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(10); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(10); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(10); + scheduledCall.invoke(); + } + }; +``` + +You are given the keys to be loaded and an optional `BatchLoaderEnvironment` for informative purposes. You can't change the list of +keys that will be loaded via this mechanism say. + +Also note, because there is a max batch size, it is possible for this scheduling to happen N times for a given `dispatch()` +call. The total set of keys will be sliced into batches themselves and then the `BatchLoaderScheduler` will be called for +each batch of keys. + +Do not assume that a single call to `dispatch()` results in a single call to `BatchLoaderScheduler`. + +This code is inspired from the scheduling code in the [reference JS implementation](https://github.com/graphql/dataloader#batch-scheduling) + +## Scheduled Registry Dispatching `ScheduledDataLoaderRegistry` is a registry that allows for dispatching to be done on a schedule. It contains a predicate that is evaluated (per data loader contained within) when `dispatchAll` is invoked. @@ -538,6 +661,154 @@ since it was last dispatched". The above acts as a kind of minimum batch depth, with a time overload. It won't dispatch if the loader depth is less than or equal to 10 but if 200ms pass it will dispatch. +## Chaining DataLoader calls + +It's natural to want to have chained `DataLoader` calls. + +```java + CompletableFuture chainedCalls = dataLoaderA.load("user1") + .thenCompose(userAsKey -> dataLoaderB.load(userAsKey)); +``` + +However, the challenge here is how to be efficient in batching terms. + +This is discussed in detail in the https://github.com/graphql-java/java-dataloader/issues/54 issue. + +Since CompletableFuture's are async and can complete at some time in the future, when is the best time to call +`dispatch` again when a load call has completed to maximize batching? + +The most naive approach is to immediately dispatch the second chained call as follows : + +```java + CompletableFuture chainedWithImmediateDispatch = dataLoaderA.load("user1") + .thenCompose(userAsKey -> { + CompletableFuture loadB = dataLoaderB.load(userAsKey); + dataLoaderB.dispatch(); + return loadB; + }); +``` + +The above will work however the window of batching together multiple calls to `dataLoaderB` will be very small and since +it will likely result in batch sizes of 1. + +This is a very difficult problem to solve because you have to balance two competing design ideals which is to maximize the +batching window of secondary calls in a small window of time so you customer requests don't take longer than necessary. + +* If the batching window is wide you will maximize the number of keys presented to a `BatchLoader` but your request latency will increase. + +* If the batching window is narrow you will reduce your request latency, but also you will reduce the number of keys presented to a `BatchLoader`. + + +### ScheduledDataLoaderRegistry ticker mode + +The `ScheduledDataLoaderRegistry` offers one solution to this called "ticker mode" where it will continually reschedule secondary +`DataLoader` calls after the initial `dispatch()` call is made. + +The batch window of time is controlled by the schedule duration setup at when the `ScheduledDataLoaderRegistry` is created. + +```java + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dataLoaderA) + .register("b", dataLoaderB) + .scheduledExecutorService(executorService) + .schedule(Duration.ofMillis(10)) + .tickerMode(true) // ticker mode is on + .build(); + + CompletableFuture chainedCalls = dataLoaderA.load("user1") + .thenCompose(userAsKey -> dataLoaderB.load(userAsKey)); + +``` +When ticker mode is on the chained dataloader calls will complete but the batching window size will depend on how quickly +the first level of `DataLoader` calls returned compared to the `schedule` of the `ScheduledDataLoaderRegistry`. + +If you use ticker mode, then you MUST `registry.close()` on the `ScheduledDataLoaderRegistry` at the end of the request (say) otherwise +it will continue to reschedule tasks to the `ScheduledExecutorService` associated with the registry. + +You will want to look at sharing the `ScheduledExecutorService` in some way between requests when creating the `ScheduledDataLoaderRegistry` +otherwise you will be creating a thread per `ScheduledDataLoaderRegistry` instance created and with enough concurrent requests +you may create too many threads. + +### ScheduledDataLoaderRegistry dispatching algorithm + +When ticker mode is **false** the `ScheduledDataLoaderRegistry` algorithm is as follows : + +* Nothing starts scheduled - some code must call `registry.dispatchAll()` a first time +* Then for every `DataLoader` in the registry + * The `DispatchPredicate` is called to test if the data loader should be dispatched + * if it returns **false** then a task is scheduled to re-evaluate this specific dataloader in the near future + * If it returns **true**, then `dataLoader.dispatch()` is called and the dataloader is not rescheduled again +* The re-evaluation tasks are run periodically according to the `registry.getScheduleDuration()` + +When ticker mode is **true** the `ScheduledDataLoaderRegistry` algorithm is as follows: + +* Nothing starts scheduled - some code must call `registry.dispatchAll()` a first time +* Then for every `DataLoader` in the registry + * The `DispatchPredicate` is called to test if the data loader should be dispatched + * if it returns **false** then a task is scheduled to re-evaluate this specific dataloader in the near future + * If it returns **true**, then `dataLoader.dispatch()` is called **and** a task is scheduled to re-evaluate this specific dataloader in the near future +* The re-evaluation tasks are run periodically according to the `registry.getScheduleDuration()` + +## Instrumenting the data loader code + +A `DataLoader` can have a `DataLoaderInstrumentation` associated with it. This callback interface is intended to provide +insight into working of the `DataLoader` such as how long it takes to run or to allow for logging of key events. + +You set the `DataLoaderInstrumentation` into the `DataLoaderOptions` at build time. + +```java + + + DataLoaderInstrumentation timingInstrumentation = new DataLoaderInstrumentation() { + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("dispatch time: %d ms", ms)); + }); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("batch loader time: %d ms", ms)); + }); + } + }; + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation).build(); + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); + +``` + +The example shows how long the overall `DataLoader` dispatch takes or how long the batch loader takes to run. + +### Instrumenting the DataLoaderRegistry + +You can also associate a `DataLoaderInstrumentation` with a `DataLoaderRegistry`. Every `DataLoader` registered will be changed so that the registry +`DataLoaderInstrumentation` is associated with it. This allows you to set just the one `DataLoaderInstrumentation` in place and it applies to all +data loaders. + +```java + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoader changedUsersDataLoader = registry.getDataLoader("users"); +``` + +The `timingInstrumentation` here will be associated with the `DataLoader` under the key `users` and the key `teams`. Note that since +DataLoader is immutable, a new changed object is created so you must use the registry to get the `DataLoader`. + ## Other information sources @@ -563,10 +834,10 @@ This library was originally written for use within a [VertX world](http://vertx. itself. All the heavy lifting has been done by this project : [vertx-dataloader](https://github.com/engagingspaces/vertx-dataloader) including the extensive testing (which itself came from Facebook). -This particular port was done to reduce the dependency on Vertx and to write a pure Java 8 implementation with no dependencies and also +This particular port was done to reduce the dependency on Vertx and to write a pure Java 11 implementation with no dependencies and also to use the more normative Java CompletableFuture. -[vertx-core](http://vertx.io/docs/vertx-core/java/) is not a lightweight library by any means so having a pure Java 8 implementation is +[vertx-core](http://vertx.io/docs/vertx-core/java/) is not a lightweight library by any means so having a pure Java 11 implementation is very desirable. diff --git a/build.gradle b/build.gradle index 8d8c392..eb57158 100644 --- a/build.gradle +++ b/build.gradle @@ -3,12 +3,21 @@ import java.text.SimpleDateFormat plugins { id 'java' id 'java-library' + id 'jvm-test-suite' id 'maven-publish' id 'signing' - id "biz.aQute.bnd.builder" version "6.2.0" - id "io.github.gradle-nexus.publish-plugin" version "1.0.0" + id 'groovy' + id 'biz.aQute.bnd.builder' version '6.2.0' + id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' + id 'com.github.ben-manes.versions' version '0.51.0' + id "me.champeau.jmh" version "0.7.3" } +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} def getDevelopmentVersion() { def output = new StringBuilder() @@ -20,25 +29,15 @@ def getDevelopmentVersion() { println "git hash is empty: error: ${error.toString()}" throw new IllegalStateException("git hash could not be determined") } - def version = new SimpleDateFormat('yyyy-MM-dd\'T\'HH-mm-ss').format(new Date()) + "-" + gitHash + def version = "0.0.0-" + new SimpleDateFormat('yyyy-MM-dd\'T\'HH-mm-ss').format(new Date()) + "-" + gitHash println "created development version: $version" version } -if (JavaVersion.current() != JavaVersion.VERSION_1_8) { - def msg = String.format("This build must be run with java 1.8 - you are running %s - gradle finds the JDK via JAVA_HOME=%s", - JavaVersion.current(), System.getenv("JAVA_HOME")) - throw new IllegalStateException(msg) -} - - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 -def slf4jVersion = '1.7.30' def releaseVersion = System.env.RELEASE_VERSION version = releaseVersion ? releaseVersion : getDevelopmentVersion() group = 'com.graphql-java' -description = 'A pure Java 8 port of Facebook Dataloader' +description = 'A pure Java 11 port of Facebook Dataloader' gradle.buildFinished { buildResult -> println "*******************************" @@ -58,47 +57,70 @@ repositories { mavenLocal() } -apply plugin: 'groovy' - jar { manifest { attributes('Automatic-Module-Name': 'org.dataloader', - '-exportcontents': 'org.dataloader.*', - '-removeheaders': 'Private-Package') + '-exportcontents': 'org.dataloader.*', + '-removeheaders': 'Private-Package') } + bnd(''' +Import-Package: org.jspecify.annotations;resolution:=optional,* +''') } dependencies { - api 'org.slf4j:slf4j-api:' + slf4jVersion - testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion - testImplementation 'junit:junit:4.12' - testImplementation 'org.awaitility:awaitility:2.0.0' - testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' + api "org.reactivestreams:reactive-streams:$reactive_streams_version" + api "org.jspecify:jspecify:1.0.0" + + // this is needed for the idea jmh plugin to work correctly + jmh 'org.openjdk.jmh:jmh-core:1.37' + jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' } task sourcesJar(type: Jar) { dependsOn classes - classifier 'sources' + archiveClassifier.set('sources') from sourceSets.main.allSource } -task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' - from javadoc.destinationDir -} - javadoc { options.encoding = 'UTF-8' } +task javadocJar(type: Jar, dependsOn: javadoc) { + archiveClassifier.set('javadoc') + from javadoc.destinationDir +} + artifacts { archives sourcesJar archives javadocJar } -test { - testLogging { - exceptionFormat = 'full' +testing { + suites { + test { + useJUnitJupiter(junit_version) + dependencies { + // Testing dependencies + implementation platform("org.junit:junit-bom:$junit_version") + implementation 'org.junit.jupiter:junit-jupiter-api' + implementation 'org.junit.jupiter:junit-jupiter-params' + implementation 'org.junit.jupiter:junit-jupiter-engine' + implementation "org.awaitility:awaitility:$awaitility_version" + implementation "org.hamcrest:hamcrest:$hamcrest_version" + implementation "io.projectreactor:reactor-core:$reactor_core_version" + implementation "com.github.ben-manes.caffeine:caffeine:$caffeine_version" + } + + targets.configureEach { + testTask.configure { + testLogging { + exceptionFormat = 'full' + } + } + } + } } } @@ -117,7 +139,7 @@ publishing { asNode().children().last() + { resolveStrategy = Closure.DELEGATE_FIRST name 'java-dataloader' - description 'A pure Java 8 port of Facebook Dataloader' + description 'A pure Java 11 port of Facebook Dataloader' url 'https://github.com/graphql-java/java-dataloader' inceptionYear '2017' @@ -162,6 +184,7 @@ nexusPublishing { } signing { + required { !project.hasProperty('publishToMavenLocal') } def signingKey = System.env.MAVEN_CENTRAL_PGP_KEY useInMemoryPgpKeys(signingKey, "") sign publishing.publications @@ -173,9 +196,15 @@ tasks.withType(PublishToMavenRepository) { dependsOn build } - -task myWrapper(type: Wrapper) { - gradleVersion = '6.6.1' - distributionUrl = "https://services.gradle.org/distributions/gradle-${gradleVersion}-all.zip" +def isNonStable = { String version -> + def stableKeyword = ['RELEASE', 'FINAL', 'GA'].any { it -> version.toUpperCase().contains(it) } + def regex = /^[0-9,.v-]+(-r)?$/ + return !stableKeyword && !(version ==~ regex) } +// https://github.com/ben-manes/gradle-versions-plugin +tasks.named("dependencyUpdates").configure { + rejectVersionIf { + isNonStable(it.candidate.version) + } +} diff --git a/gradle.properties b/gradle.properties index 0394946..428b6e2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,26 @@ +# Project-wide Gradle settings. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx4096m + +# When configured, Gradle will run in parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +org.gradle.parallel=true +org.gradle.caching=true + +# Bespoke settings. projectTitle = Java Dataloader -projectDescription = Port of Facebook Dataloader for Java \ No newline at end of file +projectDescription = Port of Facebook Dataloader for Java + +# Dependency versions. +junit_version=5.11.3 +hamcrest_version=2.2 +awaitility_version=2.0.0 +reactor_core_version=3.6.6 +caffeine_version=3.1.8 +reactive_streams_version=1.0.3 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d2880ba..e2847c8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle b/settings.gradle index e69de29..47404e7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -0,0 +1,21 @@ +plugins { + id 'com.gradle.develocity' version '3.19' + id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' +} + +develocity { + buildScan { + final def isCI = System.getenv('CI') != null; + termsOfUseUrl = "https://gradle.com/help/legal-terms-of-use" + termsOfUseAgree = "yes" + publishing.onlyIf { true } + tag(isCI ? 'CI' : 'Local') + uploadInBackground = !isCI + } +} + +dependencyResolutionManagement { + repositories { + mavenCentral() + } +} \ No newline at end of file diff --git a/src/jmh/java/performance/DataLoaderDispatchPerformance.java b/src/jmh/java/performance/DataLoaderDispatchPerformance.java new file mode 100644 index 0000000..0b4696d --- /dev/null +++ b/src/jmh/java/performance/DataLoaderDispatchPerformance.java @@ -0,0 +1,309 @@ +package performance; + +import org.dataloader.BatchLoader; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@State(Scope.Benchmark) +@Warmup(iterations = 2, time = 5) +@Measurement(iterations = 4) +@Fork(1) +public class DataLoaderDispatchPerformance { + + static Owner o1 = new Owner("O-1", "Andi", List.of("P-1", "P-2", "P-3")); + static Owner o2 = new Owner("O-2", "George", List.of("P-4", "P-5", "P-6")); + static Owner o3 = new Owner("O-3", "Peppa", List.of("P-7", "P-8", "P-9", "P-10")); + static Owner o4 = new Owner("O-4", "Alice", List.of("P-11", "P-12")); + static Owner o5 = new Owner("O-5", "Bob", List.of("P-13")); + static Owner o6 = new Owner("O-6", "Catherine", List.of("P-14", "P-15", "P-16")); + static Owner o7 = new Owner("O-7", "David", List.of("P-17")); + static Owner o8 = new Owner("O-8", "Emma", List.of("P-18", "P-19", "P-20", "P-21")); + static Owner o9 = new Owner("O-9", "Frank", List.of("P-22")); + static Owner o10 = new Owner("O-10", "Grace", List.of("P-23", "P-24")); + static Owner o11 = new Owner("O-11", "Hannah", List.of("P-25", "P-26", "P-27")); + static Owner o12 = new Owner("O-12", "Ian", List.of("P-28")); + static Owner o13 = new Owner("O-13", "Jane", List.of("P-29", "P-30")); + static Owner o14 = new Owner("O-14", "Kevin", List.of("P-31", "P-32", "P-33")); + static Owner o15 = new Owner("O-15", "Laura", List.of("P-34")); + static Owner o16 = new Owner("O-16", "Michael", List.of("P-35", "P-36")); + static Owner o17 = new Owner("O-17", "Nina", List.of("P-37", "P-38", "P-39", "P-40")); + static Owner o18 = new Owner("O-18", "Oliver", List.of("P-41")); + static Owner o19 = new Owner("O-19", "Paula", List.of("P-42", "P-43")); + static Owner o20 = new Owner("O-20", "Quinn", List.of("P-44", "P-45", "P-46")); + static Owner o21 = new Owner("O-21", "Rachel", List.of("P-47")); + static Owner o22 = new Owner("O-22", "Steve", List.of("P-48", "P-49")); + static Owner o23 = new Owner("O-23", "Tina", List.of("P-50", "P-51", "P-52")); + static Owner o24 = new Owner("O-24", "Uma", List.of("P-53")); + static Owner o25 = new Owner("O-25", "Victor", List.of("P-54", "P-55")); + static Owner o26 = new Owner("O-26", "Wendy", List.of("P-56", "P-57", "P-58")); + static Owner o27 = new Owner("O-27", "Xander", List.of("P-59")); + static Owner o28 = new Owner("O-28", "Yvonne", List.of("P-60", "P-61")); + static Owner o29 = new Owner("O-29", "Zach", List.of("P-62", "P-63", "P-64")); + static Owner o30 = new Owner("O-30", "Willy", List.of("P-65", "P-66", "P-67")); + + + static Pet p1 = new Pet("P-1", "Bella", "O-1", List.of("P-2", "P-3", "P-4")); + static Pet p2 = new Pet("P-2", "Charlie", "O-2", List.of("P-1", "P-5", "P-6")); + static Pet p3 = new Pet("P-3", "Luna", "O-3", List.of("P-1", "P-2", "P-7", "P-8")); + static Pet p4 = new Pet("P-4", "Max", "O-1", List.of("P-1", "P-9", "P-10")); + static Pet p5 = new Pet("P-5", "Lucy", "O-2", List.of("P-2", "P-6")); + static Pet p6 = new Pet("P-6", "Cooper", "O-3", List.of("P-3", "P-5", "P-7")); + static Pet p7 = new Pet("P-7", "Daisy", "O-1", List.of("P-4", "P-6", "P-8")); + static Pet p8 = new Pet("P-8", "Milo", "O-2", List.of("P-3", "P-7", "P-9")); + static Pet p9 = new Pet("P-9", "Lola", "O-3", List.of("P-4", "P-8", "P-10")); + static Pet p10 = new Pet("P-10", "Rocky", "O-1", List.of("P-4", "P-9")); + static Pet p11 = new Pet("P-11", "Buddy", "O-4", List.of("P-12")); + static Pet p12 = new Pet("P-12", "Bailey", "O-4", List.of("P-11", "P-13")); + static Pet p13 = new Pet("P-13", "Sadie", "O-5", List.of("P-12")); + static Pet p14 = new Pet("P-14", "Maggie", "O-6", List.of("P-15")); + static Pet p15 = new Pet("P-15", "Sophie", "O-6", List.of("P-14", "P-16")); + static Pet p16 = new Pet("P-16", "Chloe", "O-6", List.of("P-15")); + static Pet p17 = new Pet("P-17", "Duke", "O-7", List.of("P-18")); + static Pet p18 = new Pet("P-18", "Riley", "O-8", List.of("P-17", "P-19")); + static Pet p19 = new Pet("P-19", "Lilly", "O-8", List.of("P-18", "P-20")); + static Pet p20 = new Pet("P-20", "Zoey", "O-8", List.of("P-19")); + static Pet p21 = new Pet("P-21", "Oscar", "O-8", List.of("P-22")); + static Pet p22 = new Pet("P-22", "Toby", "O-9", List.of("P-21", "P-23")); + static Pet p23 = new Pet("P-23", "Ruby", "O-10", List.of("P-22")); + static Pet p24 = new Pet("P-24", "Milo", "O-10", List.of("P-25")); + static Pet p25 = new Pet("P-25", "Finn", "O-11", List.of("P-24", "P-26")); + static Pet p26 = new Pet("P-26", "Luna", "O-11", List.of("P-25")); + static Pet p27 = new Pet("P-27", "Ellie", "O-11", List.of("P-28")); + static Pet p28 = new Pet("P-28", "Harley", "O-12", List.of("P-27", "P-29")); + static Pet p29 = new Pet("P-29", "Penny", "O-13", List.of("P-28")); + static Pet p30 = new Pet("P-30", "Hazel", "O-13", List.of("P-31")); + static Pet p31 = new Pet("P-31", "Gus", "O-14", List.of("P-30", "P-32")); + static Pet p32 = new Pet("P-32", "Dexter", "O-14", List.of("P-31")); + static Pet p33 = new Pet("P-33", "Winnie", "O-14", List.of("P-34")); + static Pet p34 = new Pet("P-34", "Murphy", "O-15", List.of("P-33", "P-35")); + static Pet p35 = new Pet("P-35", "Moose", "O-16", List.of("P-34")); + static Pet p36 = new Pet("P-36", "Scout", "O-16", List.of("P-37")); + static Pet p37 = new Pet("P-37", "Rex", "O-17", List.of("P-36", "P-38")); + static Pet p38 = new Pet("P-38", "Coco", "O-17", List.of("P-37")); + static Pet p39 = new Pet("P-39", "Maddie", "O-17", List.of("P-40")); + static Pet p40 = new Pet("P-40", "Archie", "O-17", List.of("P-39", "P-41")); + static Pet p41 = new Pet("P-41", "Buster", "O-18", List.of("P-40")); + static Pet p42 = new Pet("P-42", "Rosie", "O-19", List.of("P-43")); + static Pet p43 = new Pet("P-43", "Molly", "O-19", List.of("P-42", "P-44")); + static Pet p44 = new Pet("P-44", "Henry", "O-20", List.of("P-43")); + static Pet p45 = new Pet("P-45", "Leo", "O-20", List.of("P-46")); + static Pet p46 = new Pet("P-46", "Jack", "O-20", List.of("P-45", "P-47")); + static Pet p47 = new Pet("P-47", "Zoe", "O-21", List.of("P-46")); + static Pet p48 = new Pet("P-48", "Lulu", "O-22", List.of("P-49")); + static Pet p49 = new Pet("P-49", "Mimi", "O-22", List.of("P-48", "P-50")); + static Pet p50 = new Pet("P-50", "Nala", "O-23", List.of("P-49")); + static Pet p51 = new Pet("P-51", "Simba", "O-23", List.of("P-52")); + static Pet p52 = new Pet("P-52", "Teddy", "O-23", List.of("P-51", "P-53")); + static Pet p53 = new Pet("P-53", "Mochi", "O-24", List.of("P-52")); + static Pet p54 = new Pet("P-54", "Oreo", "O-25", List.of("P-55")); + static Pet p55 = new Pet("P-55", "Peanut", "O-25", List.of("P-54", "P-56")); + static Pet p56 = new Pet("P-56", "Pumpkin", "O-26", List.of("P-55")); + static Pet p57 = new Pet("P-57", "Shadow", "O-26", List.of("P-58")); + static Pet p58 = new Pet("P-58", "Sunny", "O-26", List.of("P-57", "P-59")); + static Pet p59 = new Pet("P-59", "Thor", "O-27", List.of("P-58")); + static Pet p60 = new Pet("P-60", "Willow", "O-28", List.of("P-61")); + static Pet p61 = new Pet("P-61", "Zeus", "O-28", List.of("P-60", "P-62")); + static Pet p62 = new Pet("P-62", "Ace", "O-29", List.of("P-61")); + static Pet p63 = new Pet("P-63", "Blue", "O-29", List.of("P-64")); + static Pet p64 = new Pet("P-64", "Cleo", "O-29", List.of("P-63", "P-65")); + static Pet p65 = new Pet("P-65", "Dolly", "O-30", List.of("P-64")); + static Pet p66 = new Pet("P-66", "Ella", "O-30", List.of("P-67")); + static Pet p67 = new Pet("P-67", "Freddy", "O-30", List.of("P-66")); + + + static Map owners = Map.ofEntries( + Map.entry(o1.id, o1), + Map.entry(o2.id, o2), + Map.entry(o3.id, o3), + Map.entry(o4.id, o4), + Map.entry(o5.id, o5), + Map.entry(o6.id, o6), + Map.entry(o7.id, o7), + Map.entry(o8.id, o8), + Map.entry(o9.id, o9), + Map.entry(o10.id, o10), + Map.entry(o11.id, o11), + Map.entry(o12.id, o12), + Map.entry(o13.id, o13), + Map.entry(o14.id, o14), + Map.entry(o15.id, o15), + Map.entry(o16.id, o16), + Map.entry(o17.id, o17), + Map.entry(o18.id, o18), + Map.entry(o19.id, o19), + Map.entry(o20.id, o20), + Map.entry(o21.id, o21), + Map.entry(o22.id, o22), + Map.entry(o23.id, o23), + Map.entry(o24.id, o24), + Map.entry(o25.id, o25), + Map.entry(o26.id, o26), + Map.entry(o27.id, o27), + Map.entry(o28.id, o28), + Map.entry(o29.id, o29), + Map.entry(o30.id, o30) + ); + static Map pets = Map.ofEntries( + Map.entry(p1.id, p1), + Map.entry(p2.id, p2), + Map.entry(p3.id, p3), + Map.entry(p4.id, p4), + Map.entry(p5.id, p5), + Map.entry(p6.id, p6), + Map.entry(p7.id, p7), + Map.entry(p8.id, p8), + Map.entry(p9.id, p9), + Map.entry(p10.id, p10), + Map.entry(p11.id, p11), + Map.entry(p12.id, p12), + Map.entry(p13.id, p13), + Map.entry(p14.id, p14), + Map.entry(p15.id, p15), + Map.entry(p16.id, p16), + Map.entry(p17.id, p17), + Map.entry(p18.id, p18), + Map.entry(p19.id, p19), + Map.entry(p20.id, p20), + Map.entry(p21.id, p21), + Map.entry(p22.id, p22), + Map.entry(p23.id, p23), + Map.entry(p24.id, p24), + Map.entry(p25.id, p25), + Map.entry(p26.id, p26), + Map.entry(p27.id, p27), + Map.entry(p28.id, p28), + Map.entry(p29.id, p29), + Map.entry(p30.id, p30), + Map.entry(p31.id, p31), + Map.entry(p32.id, p32), + Map.entry(p33.id, p33), + Map.entry(p34.id, p34), + Map.entry(p35.id, p35), + Map.entry(p36.id, p36), + Map.entry(p37.id, p37), + Map.entry(p38.id, p38), + Map.entry(p39.id, p39), + Map.entry(p40.id, p40), + Map.entry(p41.id, p41), + Map.entry(p42.id, p42), + Map.entry(p43.id, p43), + Map.entry(p44.id, p44), + Map.entry(p45.id, p45), + Map.entry(p46.id, p46), + Map.entry(p47.id, p47), + Map.entry(p48.id, p48), + Map.entry(p49.id, p49), + Map.entry(p50.id, p50), + Map.entry(p51.id, p51), + Map.entry(p52.id, p52), + Map.entry(p53.id, p53), + Map.entry(p54.id, p54), + Map.entry(p55.id, p55), + Map.entry(p56.id, p56), + Map.entry(p57.id, p57), + Map.entry(p58.id, p58), + Map.entry(p59.id, p59), + Map.entry(p60.id, p60), + Map.entry(p61.id, p61), + Map.entry(p62.id, p62), + Map.entry(p63.id, p63), + Map.entry(p64.id, p64), + Map.entry(p65.id, p65), + Map.entry(p66.id, p66), + Map.entry(p67.id, p67) + ); + + static class Owner { + public Owner(String id, String name, List petIds) { + this.id = id; + this.name = name; + this.petIds = petIds; + } + + String id; + String name; + List petIds; + } + + static class Pet { + public Pet(String id, String name, String ownerId, List friendsIds) { + this.id = id; + this.name = name; + this.ownerId = ownerId; + this.friendsIds = friendsIds; + } + + String id; + String name; + String ownerId; + List friendsIds; + } + + + static BatchLoader ownerBatchLoader = list -> { + List collect = list.stream().map(key -> { + Owner owner = owners.get(key); + return owner; + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(collect); + }; + static BatchLoader petBatchLoader = list -> { + List collect = list.stream().map(key -> { + Pet owner = pets.get(key); + return owner; + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(collect); + }; + + + @State(Scope.Benchmark) + public static class MyState { + @Setup + public void setup() { + + } + + } + + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public void loadAndDispatch(MyState myState, Blackhole blackhole) { + DataLoader ownerDL = DataLoaderFactory.newDataLoader(ownerBatchLoader); + DataLoader petDL = DataLoaderFactory.newDataLoader(petBatchLoader); + + for (Owner owner : owners.values()) { + ownerDL.load(owner.id); + for (String petId : owner.petIds) { + petDL.load(petId); + for (String friendId : pets.get(petId).friendsIds) { + petDL.load(friendId); + } + } + } + + CompletableFuture cf1 = ownerDL.dispatch(); + CompletableFuture cf2 = petDL.dispatch(); + blackhole.consume(CompletableFuture.allOf(cf1, cf2).join()); + } + + +} diff --git a/src/jmh/java/performance/PerformanceTestingUtils.java b/src/jmh/java/performance/PerformanceTestingUtils.java new file mode 100644 index 0000000..9e05fd6 --- /dev/null +++ b/src/jmh/java/performance/PerformanceTestingUtils.java @@ -0,0 +1,84 @@ +package performance; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.Callable; + +public class PerformanceTestingUtils { + + @SuppressWarnings("UnstableApiUsage") + static String loadResource(String name) { + return asRTE(() -> { + URL resource = PerformanceTestingUtils.class.getClassLoader().getResource(name); + if (resource == null) { + throw new IllegalArgumentException("missing resource: " + name); + } + byte[] bytes; + try (InputStream inputStream = resource.openStream()) { + bytes = inputStream.readAllBytes(); + } + return new String(bytes, Charset.defaultCharset()); + }); + } + + static T asRTE(Callable callable) { + try { + return callable.call(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void runInToolingForSomeTimeThenExit(Runnable setup, Runnable r, Runnable tearDown) { + int runForMillis = getRunForMillis(); + if (runForMillis <= 0) { + System.out.print("'runForMillis' environment var is not set - continuing \n"); + return; + } + System.out.printf("Running initial code in some tooling - runForMillis=%d \n", runForMillis); + System.out.print("Get your tooling in order and press enter..."); + readLine(); + System.out.print("Lets go...\n"); + setup.run(); + + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss"); + long now, then = System.currentTimeMillis(); + do { + now = System.currentTimeMillis(); + long msLeft = runForMillis - (now - then); + System.out.printf("\t%s Running in loop... %s ms left\n", dtf.format(LocalDateTime.now()), msLeft); + r.run(); + now = System.currentTimeMillis(); + } while ((now - then) < runForMillis); + + tearDown.run(); + + System.out.printf("This ran for %d millis. Exiting...\n", System.currentTimeMillis() - then); + System.exit(0); + } + + private static void readLine() { + BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); + try { + br.readLine(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static int getRunForMillis() { + String runFor = System.getenv("runForMillis"); + try { + return Integer.parseInt(runFor); + } catch (NumberFormatException e) { + return -1; + } + } + +} diff --git a/src/main/java/org/dataloader/BatchLoader.java b/src/main/java/org/dataloader/BatchLoader.java index fed2baf..2b0c3c5 100644 --- a/src/main/java/org/dataloader/BatchLoader.java +++ b/src/main/java/org/dataloader/BatchLoader.java @@ -17,6 +17,8 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; import java.util.List; import java.util.concurrent.CompletionStage; @@ -54,7 +56,7 @@ * The back-end service returned results in a different order than we requested, likely because it was more efficient for it to * do so. Also, it omitted a result for key 6, which we may interpret as no value existing for that key. *

- * To uphold the constraints of the batch function, it must return an List of values the same length as + * To uphold the constraints of the batch function, it must return a List of values the same length as * the List of keys, and re-order them to ensure each index aligns with the original keys [ 2, 9, 6, 1 ]: * *

@@ -74,6 +76,7 @@
  */
 @FunctionalInterface
 @PublicSpi
+@NullMarked
 public interface BatchLoader {
 
     /**
diff --git a/src/main/java/org/dataloader/BatchLoaderContextProvider.java b/src/main/java/org/dataloader/BatchLoaderContextProvider.java
index d1eb1fe..702fd66 100644
--- a/src/main/java/org/dataloader/BatchLoaderContextProvider.java
+++ b/src/main/java/org/dataloader/BatchLoaderContextProvider.java
@@ -1,6 +1,7 @@
 package org.dataloader;
 
 import org.dataloader.annotations.PublicSpi;
+import org.jspecify.annotations.NullMarked;
 
 /**
  * A BatchLoaderContextProvider is used by the {@link org.dataloader.DataLoader} code to
@@ -8,9 +9,10 @@
  * case is for propagating user security credentials or database connection parameters for example.
  */
 @PublicSpi
+@NullMarked
 public interface BatchLoaderContextProvider {
     /**
      * @return a context object that may be needed in batch load calls
      */
     Object getContext();
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironment.java b/src/main/java/org/dataloader/BatchLoaderEnvironment.java
index dd2572b..6b84e70 100644
--- a/src/main/java/org/dataloader/BatchLoaderEnvironment.java
+++ b/src/main/java/org/dataloader/BatchLoaderEnvironment.java
@@ -2,6 +2,8 @@
 
 import org.dataloader.annotations.PublicApi;
 import org.dataloader.impl.Assertions;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -14,6 +16,7 @@
  * of the calling users for example or database parameters that allow the data layer call to succeed.
  */
 @PublicApi
+@NullMarked
 public class BatchLoaderEnvironment {
 
     private final Object context;
@@ -34,7 +37,7 @@ private BatchLoaderEnvironment(Object context, List keyContextsList, Map
      * @return a context object or null if there isn't one
      */
     @SuppressWarnings("unchecked")
-    public  T getContext() {
+    public  @Nullable T getContext() {
         return (T) context;
     }
 
@@ -54,7 +57,7 @@ public Map getKeyContexts() {
      * {@link org.dataloader.DataLoader#loadMany(java.util.List, java.util.List)} can be given
      * a context object when it is invoked.  A list of them is present by this method.
      *
-     * @return a list of key context objects in the order they where encountered
+     * @return a list of key context objects in the order they were encountered
      */
     public List getKeyContextsList() {
         return keyContextsList;
@@ -83,7 +86,7 @@ public  Builder keyContexts(List keys, List keyContexts) {
             Assertions.nonNull(keyContexts);
 
             Map map = new HashMap<>();
-            List list = new ArrayList<>();
+            List list = new ArrayList<>(keys.size());
             for (int i = 0; i < keys.size(); i++) {
                 K key = keys.get(i);
                 Object keyContext = null;
diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java b/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java
index fd60a14..dae7c92 100644
--- a/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java
+++ b/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java
@@ -1,6 +1,7 @@
 package org.dataloader;
 
 import org.dataloader.annotations.PublicSpi;
+import org.jspecify.annotations.NullMarked;
 
 /**
  * A BatchLoaderEnvironmentProvider is used by the {@link org.dataloader.DataLoader} code to
@@ -9,9 +10,10 @@
  * case is for propagating user security credentials or database connection parameters.
  */
 @PublicSpi
+@NullMarked
 public interface BatchLoaderEnvironmentProvider {
     /**
      * @return a {@link org.dataloader.BatchLoaderEnvironment} that may be needed in batch calls
      */
     BatchLoaderEnvironment get();
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/dataloader/BatchLoaderWithContext.java b/src/main/java/org/dataloader/BatchLoaderWithContext.java
index fbe66b0..eba26e4 100644
--- a/src/main/java/org/dataloader/BatchLoaderWithContext.java
+++ b/src/main/java/org/dataloader/BatchLoaderWithContext.java
@@ -1,6 +1,7 @@
 package org.dataloader;
 
 import org.dataloader.annotations.PublicSpi;
+import org.jspecify.annotations.NullMarked;
 
 import java.util.List;
 import java.util.concurrent.CompletionStage;
@@ -14,6 +15,7 @@
  * use this interface.
  */
 @PublicSpi
+@NullMarked
 public interface BatchLoaderWithContext {
     /**
      * Called to batch load the provided keys and return a promise to a list of values.  This default
diff --git a/src/main/java/org/dataloader/BatchPublisher.java b/src/main/java/org/dataloader/BatchPublisher.java
new file mode 100644
index 0000000..943becf
--- /dev/null
+++ b/src/main/java/org/dataloader/BatchPublisher.java
@@ -0,0 +1,41 @@
+package org.dataloader;
+
+import org.dataloader.annotations.PublicSpi;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+import org.reactivestreams.Subscriber;
+
+import java.util.List;
+
+/**
+ * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys.
+ * 

+ * The function must call the provided {@link Subscriber} to process the values it has retrieved to allow + * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available + * (rather than when all values have been retrieved). + *

+ * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. + * + * @param type parameter indicating the type of keys to use for data load requests. + * @param type parameter indicating the type of values returned + * @see BatchLoader for the non-reactive version + */ +@NullMarked +@PublicSpi +public interface BatchPublisher { + /** + * Called to batch the provided keys into a stream of values. You must provide + * the same number of values as there as keys, and they must be in the order of the keys. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + */ + void load(List keys, Subscriber subscriber); +} diff --git a/src/main/java/org/dataloader/BatchPublisherWithContext.java b/src/main/java/org/dataloader/BatchPublisherWithContext.java new file mode 100644 index 0000000..9ee010b --- /dev/null +++ b/src/main/java/org/dataloader/BatchPublisherWithContext.java @@ -0,0 +1,38 @@ +package org.dataloader; + +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; +import org.reactivestreams.Subscriber; + +import java.util.List; + +/** + * This form of {@link BatchPublisher} is given a {@link org.dataloader.BatchLoaderEnvironment} object + * that encapsulates the calling context. A typical use case is passing in security credentials or database details + * for example. + *

+ * See {@link BatchPublisher} for more details on the design invariants that you must implement in order to + * use this interface. + */ +@NullMarked +@PublicSpi +public interface BatchPublisherWithContext { + /** + * Called to batch the provided keys into a stream of values. You must provide + * the same number of values as there as keys, and they must be in the order of the keys. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. + *

+ * This is given an environment object to that maybe be useful during the call. A typical use case + * is passing in security credentials or database details for example. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + * @param environment an environment object that can help with the call + */ + void load(List keys, Subscriber subscriber, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/CacheKey.java b/src/main/java/org/dataloader/CacheKey.java index 88b5f97..c5641b1 100644 --- a/src/main/java/org/dataloader/CacheKey.java +++ b/src/main/java/org/dataloader/CacheKey.java @@ -16,6 +16,9 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + /** * Function that is invoked on input keys of type {@code K} to derive keys that are required by the {@link CacheMap} * implementation. @@ -25,6 +28,8 @@ * @author Arnold Schrijver */ @FunctionalInterface +@NullMarked +@PublicSpi public interface CacheKey { /** diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index 6d27a47..54b1b49 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -18,6 +18,8 @@ import org.dataloader.annotations.PublicSpi; import org.dataloader.impl.DefaultCacheMap; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -39,6 +41,7 @@ * @author Brad Baker */ @PublicSpi +@NullMarked public interface CacheMap { /** @@ -65,14 +68,13 @@ static CacheMap simpleMap() { /** * Gets the specified key from the cache map. *

- * May throw an exception if the key does not exists, depending on the cache map implementation that is used, - * so be sure to check {@link CacheMap#containsKey(Object)} first. + * May throw an exception if the key does not exist, depending on the cache map implementation that is used. * * @param key the key to retrieve * * @return the cached value, or {@code null} if not found (depends on cache implementation) */ - CompletableFuture get(K key); + @Nullable CompletableFuture get(K key); /** * Gets a collection of CompletableFutures from the cache map. diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index e800a55..7a50619 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -21,16 +21,22 @@ import org.dataloader.impl.CompletableFutureKit; import org.dataloader.stats.Statistics; import org.dataloader.stats.StatisticsCollector; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; +import java.util.function.Consumer; import static org.dataloader.impl.Assertions.nonNull; @@ -52,386 +58,87 @@ *

* A call to the batch loader might result in individual exception failures for item with the returned list. if * you want to capture these specific item failures then use {@link org.dataloader.Try} as a return value and - * create the data loader with {@link #newDataLoaderWithTry(BatchLoader)} form. The Try values will be interpreted + * create the data loader with {@link DataLoaderFactory#newDataLoaderWithTry(BatchLoader)} form. The Try values will be interpreted * as either success values or cause the {@link #load(Object)} promise to complete exceptionally. * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned - * * @author Arnold Schrijver * @author Brad Baker */ @PublicApi +@NullMarked public class DataLoader { + private final @Nullable String name; private final DataLoaderHelper helper; private final StatisticsCollector stats; private final CacheMap futureCache; private final ValueCache valueCache; + private final DataLoaderOptions options; + private final Object batchLoadFunction; - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoader batchLoadFunction) { - return newDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction) { - return newDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction) { - return newDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction) { - return newDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); + @VisibleForTesting + DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { + this(name, batchLoadFunction, options, Clock.systemUTC()); } - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction) { - return newMappedDataLoader(batchLoadFunction, null); - } + @VisibleForTesting + DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options, Clock clock) { + DataLoaderOptions loaderOptions = options == null ? new DataLoaderOptions() : options; + this.futureCache = determineFutureCache(loaderOptions); + this.valueCache = determineValueCache(loaderOptions); + // order of keys matter in data loader + this.stats = nonNull(loaderOptions.getStatisticsCollector()); + this.batchLoadFunction = nonNull(batchLoadFunction); + this.options = loaderOptions; + this.name = name; - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); + this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - *

- * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction) { - return newMappedDataLoaderWithTry(batchLoadFunction, null); - } - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); + @SuppressWarnings("unchecked") + private CacheMap determineFutureCache(DataLoaderOptions loaderOptions) { + return (CacheMap) loaderOptions.cacheMap().orElseGet(CacheMap::simpleMap); } - /** - * Creates new DataLoader with the specified mapped batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction) { - return newMappedDataLoader(batchLoadFunction, null); + @SuppressWarnings("unchecked") + private ValueCache determineValueCache(DataLoaderOptions loaderOptions) { + return (ValueCache) loaderOptions.valueCache().orElseGet(ValueCache::defaultValueCache); } /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead + * @return the name of the DataLoader which can be null */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); + public @Nullable String getName() { + return name; } /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @deprecated use {@link DataLoaderFactory} instead + * @return the options used to build this {@link DataLoader} */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction) { - return newMappedDataLoaderWithTry(batchLoadFunction, null); + public DataLoaderOptions getOptions() { + return options; } /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * - * @return a new DataLoader - * - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead + * @return the batch load interface used to build this {@link DataLoader} */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); + public Object getBatchLoadFunction() { + return batchLoadFunction; } /** - * Creates a new data loader with the provided batch load function, and default options. - * - * @param batchLoadFunction the batch load function to use + * This allows you to change the current {@link DataLoader} and turn it into a new one * - * @deprecated use {@link DataLoaderFactory} instead + * @param builderConsumer the {@link DataLoaderFactory.Builder} consumer for changing the {@link DataLoader} + * @return a newly built {@link DataLoader} instance */ - @Deprecated - public DataLoader(BatchLoader batchLoadFunction) { - this((Object) batchLoadFunction, null); - } - - /** - * Creates a new data loader with the provided batch load function and options. - * - * @param batchLoadFunction the batch load function to use - * @param options the batch load options - * - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public DataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { - this((Object) batchLoadFunction, options); - } - - @VisibleForTesting - DataLoader(Object batchLoadFunction, DataLoaderOptions options) { - this(batchLoadFunction, options, Clock.systemUTC()); - } - - @VisibleForTesting - DataLoader(Object batchLoadFunction, DataLoaderOptions options, Clock clock) { - DataLoaderOptions loaderOptions = options == null ? new DataLoaderOptions() : options; - this.futureCache = determineFutureCache(loaderOptions); - this.valueCache = determineValueCache(loaderOptions); - // order of keys matter in data loader - this.stats = nonNull(loaderOptions.getStatisticsCollector()); - - this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); - } - - - @SuppressWarnings("unchecked") - private CacheMap determineFutureCache(DataLoaderOptions loaderOptions) { - return (CacheMap) loaderOptions.cacheMap().orElseGet(CacheMap::simpleMap); - } - - @SuppressWarnings("unchecked") - private ValueCache determineValueCache(DataLoaderOptions loaderOptions) { - return (ValueCache) loaderOptions.valueCache().orElseGet(ValueCache::defaultValueCache); + public DataLoader transform(Consumer> builderConsumer) { + DataLoaderFactory.Builder builder = DataLoaderFactory.builder(this); + builderConsumer.accept(builder); + return builder.build(); } /** @@ -460,7 +167,6 @@ public Duration getTimeSinceDispatch() { * and returned from cache). * * @param key the key to load - * * @return the future of the value */ public CompletableFuture load(K key) { @@ -471,14 +177,13 @@ public CompletableFuture load(K key) { * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call or empty if not call has been made for that key. *

* If you do get a present CompletableFuture it does not mean it has been dispatched and completed yet. It just means - * its at least pending and in cache. + * it's at least pending and in cache. *

* If caching is disabled there will never be a present Optional returned. *

- * NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. + * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check - * * @return an Optional to the future of the value */ public Optional> getIfPresent(K key) { @@ -494,10 +199,9 @@ public Optional> getIfPresent(K key) { *

* If caching is disabled there will never be a present Optional returned. *

- * NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. + * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check - * * @return an Optional to the future of the value */ public Optional> getIfCompleted(K key) { @@ -517,11 +221,10 @@ public Optional> getIfCompleted(K key) { * * @param key the key to load * @param keyContext a context object that is specific to this key - * * @return the future of the value */ - public CompletableFuture load(K key, Object keyContext) { - return helper.load(key, keyContext); + public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { + return helper.load(nonNull(key), keyContext); } /** @@ -533,7 +236,6 @@ public CompletableFuture load(K key, Object keyContext) { * and returned from cache). * * @param keys the list of keys to load - * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys) { @@ -553,7 +255,6 @@ public CompletableFuture> loadMany(List keys) { * * @param keys the list of keys to load * @param keyContexts the list of key calling context objects - * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys, List keyContexts) { @@ -561,7 +262,7 @@ public CompletableFuture> loadMany(List keys, List keyContext nonNull(keyContexts); synchronized (this) { - List> collect = new ArrayList<>(); + List> collect = new ArrayList<>(keys.size()); for (int i = 0; i < keys.size(); i++) { K key = keys.get(i); Object keyContext = null; @@ -574,6 +275,34 @@ public CompletableFuture> loadMany(List keys, List keyContext } } + /** + * Requests to load the map of data provided by the specified keys asynchronously, and returns a composite future + * of the resulting values. + *

+ * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to + * start batch execution. If you forget this call the future will never be completed (unless already completed, + * and returned from cache). + *

+ * The key context object may be useful in the batch loader interfaces such as {@link org.dataloader.BatchLoaderWithContext} or + * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. + * + * @param keysAndContexts the map of keys to their respective contexts + * @return the composite future of the map of keys and values + */ + public CompletableFuture> loadMany(Map keysAndContexts) { + nonNull(keysAndContexts); + + synchronized (this) { + Map> collect = new HashMap<>(keysAndContexts.size()); + for (Map.Entry entry : keysAndContexts.entrySet()) { + K key = entry.getKey(); + Object keyContext = entry.getValue(); + collect.put(key, load(key, keyContext)); + } + return CompletableFutureKit.allOf(collect); + } + } + /** * Dispatches the queued load requests to the batch execution function and returns a promise of the result. *

@@ -630,7 +359,6 @@ public int dispatchDepth() { * on the next load request. * * @param key the key to remove - * * @return the data loader for fluent coding */ public DataLoader clear(K key) { @@ -644,7 +372,6 @@ public DataLoader clear(K key) { * * @param key the key to remove * @param handler a handler that will be called after the async remote clear completes - * * @return the data loader for fluent coding */ public DataLoader clear(K key, BiConsumer handler) { @@ -670,7 +397,6 @@ public DataLoader clearAll() { * Clears the entire cache map of the loader, and of the cached value store. * * @param handler a handler that will be called after the async remote clear all completes - * * @return the data loader for fluent coding */ public DataLoader clearAll(BiConsumer handler) { @@ -688,7 +414,6 @@ public DataLoader clearAll(BiConsumer handler) { * * @param key the key * @param value the value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, V value) { @@ -700,7 +425,6 @@ public DataLoader prime(K key, V value) { * * @param key the key * @param error the exception to prime instead of a value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, Exception error) { @@ -714,7 +438,6 @@ public DataLoader prime(K key, Exception error) { * * @param key the key * @param value the value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, CompletableFuture value) { @@ -734,7 +457,6 @@ public DataLoader prime(K key, CompletableFuture value) { * If no cache key function is present in {@link DataLoaderOptions}, then the returned value equals the input key. * * @param key the input key - * * @return the cache key after the input is transformed with the cache key function */ public Object getCacheKey(K key) { @@ -753,6 +475,7 @@ public Statistics getStatistics() { /** * Gets the cacheMap associated with this data loader passed in via {@link DataLoaderOptions#cacheMap()} + * * @return the cacheMap of this data loader */ public CacheMap getCacheMap() { @@ -762,10 +485,18 @@ public CacheMap getCacheMap() { /** * Gets the valueCache associated with this data loader passed in via {@link DataLoaderOptions#valueCache()} + * * @return the valueCache of this data loader */ public ValueCache getValueCache() { return valueCache; } + @Override + public String toString() { + return "DataLoader{" + + "name='" + name + '\'' + + ", stats=" + stats + + '}'; + } } diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index 0f910c1..509c16a 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -1,6 +1,9 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.jspecify.annotations.Nullable; + +import static org.dataloader.impl.Assertions.nonNull; /** * A factory class to create {@link DataLoader}s @@ -16,13 +19,26 @@ public class DataLoaderFactory { * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoader batchLoadFunction) { return newDataLoader(batchLoadFunction, null); } + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(String name, BatchLoader batchLoadFunction) { + return newDataLoader(name, batchLoadFunction, null); + } + /** * Creates new DataLoader with the specified batch loader function with the provided options * @@ -30,11 +46,24 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(String name, BatchLoader batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -42,7 +71,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -51,7 +80,6 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction) { @@ -67,13 +95,28 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newDataLoaderWithTry(String name, BatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -83,7 +126,6 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction) { @@ -97,11 +139,24 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(String name, BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -109,7 +164,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -118,7 +173,6 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction) { @@ -134,13 +188,28 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newDataLoaderWithTry(String name, BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -150,7 +219,6 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction) { @@ -164,11 +232,23 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type * @return a new DataLoader */ - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + public static DataLoader newMappedDataLoader(String name, MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -176,7 +256,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader - * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -186,7 +266,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction) { @@ -202,13 +281,28 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedDataLoaderWithTry(String name, MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -218,7 +312,6 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction) { @@ -232,11 +325,24 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedDataLoader(String name, MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -244,7 +350,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -253,7 +359,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction) { @@ -269,16 +374,507 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedDataLoaderWithTry(String name, MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction) { + return newPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(String name, BatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction) { + return newPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newPublisherDataLoaderWithTry(String name, BatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction) { + return newPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(String name, BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } - static DataLoader mkDataLoader(Object batchLoadFunction, DataLoaderOptions options) { - return new DataLoader<>(batchLoadFunction, options); + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction) { + return newPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newPublisherDataLoaderWithTry(BatchPublisher) + */ + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newPublisherDataLoaderWithTry(BatchPublisher) + */ + public static DataLoader newPublisherDataLoaderWithTry(String name, BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction) { + return newMappedPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(String name, MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction) { + return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(String name, MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction) { + return newMappedPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(String name, MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction) { + return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(String name, MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); + } + + static DataLoader mkDataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { + return new DataLoader<>(name, batchLoadFunction, options); + } + + /** + * Return a new {@link Builder} of a data loader. + * + * @param the key type + * @param the value type + * @return a new {@link Builder} of a data loader + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Return a new {@link Builder} of a data loader using the specified one as a template. + * + * @param the key type + * @param the value type + * @param dataLoader the {@link DataLoader} to copy values from into the builder + * @return a new {@link Builder} of a data loader + */ + public static Builder builder(DataLoader dataLoader) { + return new Builder<>(dataLoader); + } + + /** + * A builder of {@link DataLoader}s + * + * @param the key type + * @param the value type + */ + public static class Builder { + String name; + Object batchLoadFunction; + DataLoaderOptions options = DataLoaderOptions.newDefaultOptions(); + + Builder() { + } + + Builder(DataLoader dataLoader) { + this.name = dataLoader.getName(); + this.batchLoadFunction = dataLoader.getBatchLoadFunction(); + this.options = dataLoader.getOptions(); + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder options(DataLoaderOptions options) { + this.options = options; + return this; + } + + public Builder batchLoadFunction(Object batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder batchLoader(BatchLoader batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder batchLoader(BatchLoaderWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedBatchLoader(MappedBatchLoader batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedBatchLoader(MappedBatchLoaderWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder publisherBatchLoader(BatchPublisher batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder publisherBatchLoader(BatchPublisherWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedPublisherBatchLoader(MappedBatchPublisher batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedPublisherBatchLoader(MappedBatchPublisherWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public DataLoader build() { + return mkDataLoader(name, batchLoadFunction, options); + } } } + diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 883189c..7858780 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -3,12 +3,17 @@ import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationContext; +import org.dataloader.reactive.ReactiveSupport; +import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; import org.dataloader.stats.context.IncrementBatchLoadCountByStatisticsContext; import org.dataloader.stats.context.IncrementBatchLoadExceptionCountStatisticsContext; import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; +import org.reactivestreams.Subscriber; import java.time.Clock; import java.time.Instant; @@ -31,10 +36,11 @@ import static java.util.stream.Collectors.toList; import static org.dataloader.impl.Assertions.assertState; import static org.dataloader.impl.Assertions.nonNull; +import static org.dataloader.instrumentation.DataLoaderInstrumentationHelper.ctxOrNoopCtx; /** - * This helps break up the large DataLoader class functionality and it contains the logic to dispatch the - * promises on behalf of its peer dataloader + * This helps break up the large DataLoader class functionality, and it contains the logic to dispatch the + * promises on behalf of its peer dataloader. * * @param the type of keys * @param the type of values @@ -109,9 +115,13 @@ Optional> getIfPresent(K key) { boolean cachingEnabled = loaderOptions.cachingEnabled(); if (cachingEnabled) { Object cacheKey = getCacheKey(nonNull(key)); - if (futureCache.containsKey(cacheKey)) { - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); - return Optional.of(futureCache.get(cacheKey)); + try { + CompletableFuture cacheValue = futureCache.get(cacheKey); + if (cacheValue != null) { + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); + return Optional.of(cacheValue); + } + } catch (Exception ignored) { } } } @@ -138,12 +148,16 @@ CompletableFuture load(K key, Object loadContext) { boolean cachingEnabled = loaderOptions.cachingEnabled(); stats.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(key, loadContext)); - + DataLoaderInstrumentationContext ctx = ctxOrNoopCtx(instrumentation().beginLoad(dataLoader, key,loadContext)); + CompletableFuture cf; if (cachingEnabled) { - return loadFromCache(key, loadContext, batchingEnabled); + cf = loadFromCache(key, loadContext, batchingEnabled); } else { - return queueOrInvokeLoader(key, loadContext, batchingEnabled, false); + cf = queueOrInvokeLoader(key, loadContext, batchingEnabled, false); } + ctx.onDispatched(); + cf.whenComplete(ctx::onCompleted); + return cf; } } @@ -160,13 +174,25 @@ Object getCacheKeyWithContext(K key, Object context) { } DispatchResult dispatch() { + DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); + boolean batchingEnabled = loaderOptions.batchingEnabled(); - // - // we copy the pre-loaded set of futures ready for dispatch - final List keys = new ArrayList<>(); - final List callContexts = new ArrayList<>(); - final List> queuedFutures = new ArrayList<>(); + final List keys; + final List callContexts; + final List> queuedFutures; synchronized (dataLoader) { + int queueSize = loaderQueue.size(); + if (queueSize == 0) { + lastDispatchTime.set(now()); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, emptyDispatchResult()); + } + + // we copy the pre-loaded set of futures ready for dispatch + keys = new ArrayList<>(queueSize); + callContexts = new ArrayList<>(queueSize); + queuedFutures = new ArrayList<>(queueSize); + loaderQueue.forEach(entry -> { keys.add(entry.getKey()); queuedFutures.add(entry.getValue()); @@ -175,8 +201,9 @@ DispatchResult dispatch() { loaderQueue.clear(); lastDispatchTime.set(now()); } - if (!batchingEnabled || keys.isEmpty()) { - return new DispatchResult<>(completedFuture(emptyList()), 0); + if (!batchingEnabled) { + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, emptyDispatchResult()); } final int totalEntriesHandled = keys.size(); // @@ -197,15 +224,23 @@ DispatchResult dispatch() { } else { futureList = dispatchQueueBatch(keys, callContexts, queuedFutures); } - return new DispatchResult<>(futureList, totalEntriesHandled); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, new DispatchResult<>(futureList, totalEntriesHandled)); + } + + private DispatchResult endDispatchCtx(DataLoaderInstrumentationContext> instrCtx, DispatchResult dispatchResult) { + // once the CF completes, we can tell the instrumentation + dispatchResult.getPromisedResults() + .whenComplete((result, throwable) -> instrCtx.onCompleted(dispatchResult, throwable)); + return dispatchResult; } private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List> queuedFutures, List callContexts, int maxBatchSize) { // the number of keys is > than what the batch loader function can accept // so make multiple calls to the loader - List>> allBatches = new ArrayList<>(); int len = keys.size(); int batchCount = (int) Math.ceil(len / (double) maxBatchSize); + List>> allBatches = new ArrayList<>(batchCount); for (int i = 0; i < batchCount; i++) { int fromIndex = i * maxBatchSize; @@ -229,10 +264,14 @@ private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List< @SuppressWarnings("unchecked") private CompletableFuture> dispatchQueueBatch(List keys, List callContexts, List> queuedFutures) { stats.incrementBatchLoadCountBy(keys.size(), new IncrementBatchLoadCountByStatisticsContext<>(keys, callContexts)); - CompletableFuture> batchLoad = invokeLoader(keys, callContexts, loaderOptions.cachingEnabled()); + CompletableFuture> batchLoad = invokeLoader(keys, callContexts, queuedFutures, loaderOptions.cachingEnabled()); return batchLoad .thenApply(values -> { assertResultSize(keys, values); + if (isPublisher() || isMappedPublisher()) { + // We have already completed the queued futures by the time the overall batchLoad future has completed. + return values; + } List clearCacheKeys = new ArrayList<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { @@ -286,9 +325,9 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { if (keys.isEmpty()) { return; } - // by default we don't clear the cached view of this entry to avoid - // frequently loading the same error. This works for short lived request caches - // but might work against long lived caches. Hence we have an option that allows + // by default, we don't clear the cached view of this entry to avoid + // frequently loading the same error. This works for short-lived request caches + // but might work against long-lived caches. Hence, we have an option that allows // it to be cleared if (!loaderOptions.cachingExceptionsEnabled()) { keys.forEach(dataLoader::clear); @@ -299,10 +338,14 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { private CompletableFuture loadFromCache(K key, Object loadContext, boolean batchingEnabled) { final Object cacheKey = loadContext == null ? getCacheKey(key) : getCacheKeyWithContext(key, loadContext); - if (futureCache.containsKey(cacheKey)) { - // We already have a promise for this key, no need to check value cache or queue up load - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - return futureCache.get(cacheKey); + try { + CompletableFuture cacheValue = futureCache.get(cacheKey); + if (cacheValue != null) { + // We already have a promise for this key, no need to check value cache or queue up load + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + return cacheValue; + } + } catch (Exception ignored) { } CompletableFuture loadCallFuture = queueOrInvokeLoader(key, loadContext, batchingEnabled, true); @@ -326,14 +369,15 @@ private CompletableFuture queueOrInvokeLoader(K key, Object loadContext, bool CompletableFuture invokeLoaderImmediately(K key, Object keyContext, boolean cachingEnabled) { List keys = singletonList(key); List keyContexts = singletonList(keyContext); - return invokeLoader(keys, keyContexts, cachingEnabled) + List> queuedFutures = singletonList(new CompletableFuture<>()); + return invokeLoader(keys, keyContexts, queuedFutures, cachingEnabled) .thenApply(list -> list.get(0)) .toCompletableFuture(); } - CompletableFuture> invokeLoader(List keys, List keyContexts, boolean cachingEnabled) { + CompletableFuture> invokeLoader(List keys, List keyContexts, List> queuedFutures, boolean cachingEnabled) { if (!cachingEnabled) { - return invokeLoader(keys, keyContexts); + return invokeLoader(keys, keyContexts, queuedFutures); } CompletableFuture>> cacheCallCF = getFromValueCache(keys); return cacheCallCF.thenCompose(cachedValues -> { @@ -344,6 +388,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, List missedKeyIndexes = new ArrayList<>(); List missedKeys = new ArrayList<>(); List missedKeyContexts = new ArrayList<>(); + List> missedQueuedFutures = new ArrayList<>(); // if they return a ValueCachingNotSupported exception then we insert this special marker value, and it // means it's a total miss, we need to get all these keys via the batch loader @@ -353,6 +398,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, missedKeyIndexes.add(i); missedKeys.add(keys.get(i)); missedKeyContexts.add(keyContexts.get(i)); + missedQueuedFutures.add(queuedFutures.get(i)); } } else { assertState(keys.size() == cachedValues.size(), () -> "The size of the cached values MUST be the same size as the key list"); @@ -363,6 +409,9 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, missedKeyIndexes.add(i); missedKeys.add(keys.get(i)); missedKeyContexts.add(keyContexts.get(i)); + missedQueuedFutures.add(queuedFutures.get(i)); + } else { + queuedFutures.get(i).complete(cacheGet.get()); } } } @@ -374,10 +423,10 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, return completedFuture(assembledValues); } else { // - // we missed some of the keys from cache, so send them to the batch loader + // we missed some keys from cache, so send them to the batch loader // and then fill in their values // - CompletableFuture> batchLoad = invokeLoader(missedKeys, missedKeyContexts); + CompletableFuture> batchLoad = invokeLoader(missedKeys, missedKeyContexts, missedQueuedFutures); return batchLoad.thenCompose(missedValues -> { assertResultSize(missedKeys, missedValues); @@ -396,31 +445,54 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, }); } + CompletableFuture> invokeLoader(List keys, List keyContexts, List> queuedFutures) { + Object context = loaderOptions.getBatchLoaderContextProvider().getContext(); + BatchLoaderEnvironment environment = BatchLoaderEnvironment.newBatchLoaderEnvironment() + .context(context).keyContexts(keys, keyContexts).build(); + + DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginBatchLoader(dataLoader, keys, environment)); - CompletableFuture> invokeLoader(List keys, List keyContexts) { CompletableFuture> batchLoad; try { - Object context = loaderOptions.getBatchLoaderContextProvider().getContext(); - BatchLoaderEnvironment environment = BatchLoaderEnvironment.newBatchLoaderEnvironment() - .context(context).keyContexts(keys, keyContexts).build(); if (isMapLoader()) { batchLoad = invokeMapBatchLoader(keys, environment); + } else if (isPublisher()) { + batchLoad = invokeBatchPublisher(keys, keyContexts, queuedFutures, environment); + } else if (isMappedPublisher()) { + batchLoad = invokeMappedBatchPublisher(keys, keyContexts, queuedFutures, environment); } else { batchLoad = invokeListBatchLoader(keys, environment); } + instrCtx.onDispatched(); } catch (Exception e) { + instrCtx.onDispatched(); batchLoad = CompletableFutureKit.failedFuture(e); } + batchLoad.whenComplete(instrCtx::onCompleted); return batchLoad; } + @SuppressWarnings("unchecked") private CompletableFuture> invokeListBatchLoader(List keys, BatchLoaderEnvironment environment) { CompletionStage> loadResult; + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof BatchLoaderWithContext) { - loadResult = ((BatchLoaderWithContext) batchLoadFunction).load(keys, environment); + BatchLoaderWithContext loadFunction = (BatchLoaderWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchLoaderCall loadCall = () -> loadFunction.load(keys, environment); + loadResult = batchLoaderScheduler.scheduleBatchLoader(loadCall, keys, environment); + } else { + loadResult = loadFunction.load(keys, environment); + } } else { - loadResult = ((BatchLoader) batchLoadFunction).load(keys); + BatchLoader loadFunction = (BatchLoader) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchLoaderCall loadCall = () -> loadFunction.load(keys); + loadResult = batchLoaderScheduler.scheduleBatchLoader(loadCall, keys, null); + } else { + loadResult = loadFunction.load(keys); + } } return nonNull(loadResult, () -> "Your batch loader function MUST return a non null CompletionStage").toCompletableFuture(); } @@ -434,14 +506,27 @@ private CompletableFuture> invokeListBatchLoader(List keys, BatchLoad private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoaderEnvironment environment) { CompletionStage> loadResult; Set setOfKeys = new LinkedHashSet<>(keys); + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchLoaderWithContext) { - loadResult = ((MappedBatchLoaderWithContext) batchLoadFunction).load(setOfKeys, environment); + MappedBatchLoaderWithContext loadFunction = (MappedBatchLoaderWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledMappedBatchLoaderCall loadCall = () -> loadFunction.load(setOfKeys, environment); + loadResult = batchLoaderScheduler.scheduleMappedBatchLoader(loadCall, keys, environment); + } else { + loadResult = loadFunction.load(setOfKeys, environment); + } } else { - loadResult = ((MappedBatchLoader) batchLoadFunction).load(setOfKeys); + MappedBatchLoader loadFunction = (MappedBatchLoader) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledMappedBatchLoaderCall loadCall = () -> loadFunction.load(setOfKeys); + loadResult = batchLoaderScheduler.scheduleMappedBatchLoader(loadCall, keys, null); + } else { + loadResult = loadFunction.load(setOfKeys); + } } CompletableFuture> mapBatchLoad = nonNull(loadResult, () -> "Your batch loader function MUST return a non null CompletionStage").toCompletableFuture(); return mapBatchLoad.thenApply(map -> { - List values = new ArrayList<>(); + List values = new ArrayList<>(keys.size()); for (K key : keys) { V value = map.get(key); values.add(value); @@ -450,10 +535,76 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade }); } + private CompletableFuture> invokeBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + CompletableFuture> loadResult = new CompletableFuture<>(); + Subscriber subscriber = ReactiveSupport.batchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); + if (batchLoadFunction instanceof BatchPublisherWithContext) { + //noinspection unchecked + BatchPublisherWithContext loadFunction = (BatchPublisherWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, environment); + } else { + loadFunction.load(keys, subscriber, environment); + } + } else { + //noinspection unchecked + BatchPublisher loadFunction = (BatchPublisher) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, null); + } else { + loadFunction.load(keys, subscriber); + } + } + return loadResult; + } + + private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + CompletableFuture> loadResult = new CompletableFuture<>(); + Subscriber> subscriber = ReactiveSupport.mappedBatchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + Set setOfKeys = new LinkedHashSet<>(keys); + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); + if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { + //noinspection unchecked + MappedBatchPublisherWithContext loadFunction = (MappedBatchPublisherWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, environment); + } else { + loadFunction.load(keys, subscriber, environment); + } + } else { + //noinspection unchecked + MappedBatchPublisher loadFunction = (MappedBatchPublisher) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(setOfKeys, subscriber); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, null); + } else { + loadFunction.load(setOfKeys, subscriber); + } + } + return loadResult; + } + private boolean isMapLoader() { return batchLoadFunction instanceof MappedBatchLoader || batchLoadFunction instanceof MappedBatchLoaderWithContext; } + private boolean isPublisher() { + return batchLoadFunction instanceof BatchPublisher; + } + + private boolean isMappedPublisher() { + return batchLoadFunction instanceof MappedBatchPublisher; + } + + private DataLoaderInstrumentation instrumentation() { + return loaderOptions.getInstrumentation(); + } + int dispatchDepth() { synchronized (dataLoader) { return loaderQueue.size(); @@ -497,4 +648,30 @@ private CompletableFuture> setToValueCache(List assembledValues, List } return CompletableFuture.completedFuture(assembledValues); } + + private static final DispatchResult EMPTY_DISPATCH_RESULT = new DispatchResult<>(completedFuture(emptyList()), 0); + + @SuppressWarnings("unchecked") // Casting to any type is safe since the underlying list is empty + private static DispatchResult emptyDispatchResult() { + return (DispatchResult) EMPTY_DISPATCH_RESULT; + } + + private ReactiveSupport.HelperIntegration helperIntegration() { + return new ReactiveSupport.HelperIntegration<>() { + @Override + public StatisticsCollector getStats() { + return stats; + } + + @Override + public void clearCacheView(K key) { + dataLoader.clear(key); + } + + @Override + public void clearCacheEntriesOnExceptions(List keys) { + possiblyClearCacheEntriesOnExceptions(keys); + } + }; + } } diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index 4c79296..f7c006f 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -17,17 +17,22 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; -import org.dataloader.impl.Assertions; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; +import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.NoOpStatisticsCollector; import org.dataloader.stats.StatisticsCollector; +import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.dataloader.impl.Assertions.nonNull; /** - * Configuration options for {@link DataLoader} instances. + * Configuration options for {@link DataLoader} instances. This is an immutable class so each time + * you change a value it returns a new object. * * @author Arnold Schrijver */ @@ -35,17 +40,21 @@ public class DataLoaderOptions { private static final BatchLoaderContextProvider NULL_PROVIDER = () -> null; + private static final Supplier NOOP_COLLECTOR = NoOpStatisticsCollector::new; + private static final ValueCacheOptions DEFAULT_VALUE_CACHE_OPTIONS = ValueCacheOptions.newOptions(); - private boolean batchingEnabled; - private boolean cachingEnabled; - private boolean cachingExceptionsEnabled; - private CacheKey cacheKeyFunction; - private CacheMap cacheMap; - private ValueCache valueCache; - private int maxBatchSize; - private Supplier statisticsCollector; - private BatchLoaderContextProvider environmentProvider; - private ValueCacheOptions valueCacheOptions; + private final boolean batchingEnabled; + private final boolean cachingEnabled; + private final boolean cachingExceptionsEnabled; + private final CacheKey cacheKeyFunction; + private final CacheMap cacheMap; + private final ValueCache valueCache; + private final int maxBatchSize; + private final Supplier statisticsCollector; + private final BatchLoaderContextProvider environmentProvider; + private final ValueCacheOptions valueCacheOptions; + private final BatchLoaderScheduler batchLoaderScheduler; + private final DataLoaderInstrumentation instrumentation; /** * Creates a new data loader options with default settings. @@ -54,85 +63,108 @@ public DataLoaderOptions() { batchingEnabled = true; cachingEnabled = true; cachingExceptionsEnabled = true; + cacheKeyFunction = null; + cacheMap = null; + valueCache = null; maxBatchSize = -1; - statisticsCollector = NoOpStatisticsCollector::new; + statisticsCollector = NOOP_COLLECTOR; environmentProvider = NULL_PROVIDER; - valueCacheOptions = ValueCacheOptions.newOptions(); + valueCacheOptions = DEFAULT_VALUE_CACHE_OPTIONS; + batchLoaderScheduler = null; + instrumentation = DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION; } - /** - * Clones the provided data loader options. - * - * @param other the other options instance - */ - public DataLoaderOptions(DataLoaderOptions other) { - nonNull(other); - this.batchingEnabled = other.batchingEnabled; - this.cachingEnabled = other.cachingEnabled; - this.cachingExceptionsEnabled = other.cachingExceptionsEnabled; - this.cacheKeyFunction = other.cacheKeyFunction; - this.cacheMap = other.cacheMap; - this.valueCache = other.valueCache; - this.maxBatchSize = other.maxBatchSize; - this.statisticsCollector = other.statisticsCollector; - this.environmentProvider = other.environmentProvider; - this.valueCacheOptions = other.valueCacheOptions; + private DataLoaderOptions(Builder builder) { + this.batchingEnabled = builder.batchingEnabled; + this.cachingEnabled = builder.cachingEnabled; + this.cachingExceptionsEnabled = builder.cachingExceptionsEnabled; + this.cacheKeyFunction = builder.cacheKeyFunction; + this.cacheMap = builder.cacheMap; + this.valueCache = builder.valueCache; + this.maxBatchSize = builder.maxBatchSize; + this.statisticsCollector = builder.statisticsCollector; + this.environmentProvider = builder.environmentProvider; + this.valueCacheOptions = builder.valueCacheOptions; + this.batchLoaderScheduler = builder.batchLoaderScheduler; + this.instrumentation = builder.instrumentation; } /** * @return a new default data loader options that you can then customize */ - public static DataLoaderOptions newOptions() { + public static DataLoaderOptions newDefaultOptions() { return new DataLoaderOptions(); } /** - * Option that determines whether to use batching (the default), or not. - * - * @return {@code true} when batching is enabled, {@code false} otherwise + * @return a new default data loader options builder that you can then customize */ - public boolean batchingEnabled() { - return batchingEnabled; + public static DataLoaderOptions.Builder newOptions() { + return new Builder(); } /** - * Sets the option that determines whether batch loading is enabled. - * - * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise + * Copies the options into a new builder * - * @return the data loader options for fluent coding + * @return a new default data loader options builder that you can then customize */ - public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { - this.batchingEnabled = batchingEnabled; - return this; + public static DataLoaderOptions.Builder newOptions(DataLoaderOptions otherOptions) { + return new Builder(otherOptions); } /** - * Option that determines whether to use caching of futures (the default), or not. + * Will transform the current options in to a builder ands allow you to build a new set of options * - * @return {@code true} when caching is enabled, {@code false} otherwise + * @param builderConsumer the consumer of a builder that has this objects starting values + * @return a new {@link DataLoaderOptions} object */ - public boolean cachingEnabled() { - return cachingEnabled; + public DataLoaderOptions transform(Consumer builderConsumer) { + Builder builder = new Builder(this); + builderConsumer.accept(builder); + return builder.build(); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + DataLoaderOptions that = (DataLoaderOptions) o; + return batchingEnabled == that.batchingEnabled + && cachingEnabled == that.cachingEnabled + && cachingExceptionsEnabled == that.cachingExceptionsEnabled + && maxBatchSize == that.maxBatchSize + && Objects.equals(cacheKeyFunction, that.cacheKeyFunction) && + Objects.equals(cacheMap, that.cacheMap) && + Objects.equals(valueCache, that.valueCache) && + Objects.equals(statisticsCollector, that.statisticsCollector) && + Objects.equals(environmentProvider, that.environmentProvider) && + Objects.equals(valueCacheOptions, that.valueCacheOptions) && + Objects.equals(batchLoaderScheduler, that.batchLoaderScheduler); } + /** - * Sets the option that determines whether caching is enabled. + * Option that determines whether to use batching (the default), or not. * - * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise + * @return {@code true} when batching is enabled, {@code false} otherwise + */ + public boolean batchingEnabled() { + return batchingEnabled; + } + + /** + * Option that determines whether to use caching of futures (the default), or not. * - * @return the data loader options for fluent coding + * @return {@code true} when caching is enabled, {@code false} otherwise */ - public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { - this.cachingEnabled = cachingEnabled; - return this; + public boolean cachingEnabled() { + return cachingEnabled; } /** * Option that determines whether to cache exceptional values (the default), or not. - * - * For short lived caches (that is request caches) it makes sense to cache exceptions since - * its likely the key is still poisoned. However if you have long lived caches, then it may make + *

+ * For short-lived caches (that is request caches) it makes sense to cache exceptions since + * it's likely the key is still poisoned. However, if you have long-lived caches, then it may make * sense to set this to false since the downstream system may have recovered from its failure * mode. * @@ -142,18 +174,6 @@ public boolean cachingExceptionsEnabled() { return cachingExceptionsEnabled; } - /** - * Sets the option that determines whether exceptional values are cachedis enabled. - * - * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { - this.cachingExceptionsEnabled = cachingExceptionsEnabled; - return this; - } - /** * Gets an (optional) function to invoke for creation of the cache key, if caching is enabled. *

@@ -165,18 +185,6 @@ public Optional cacheKeyFunction() { return Optional.ofNullable(cacheKeyFunction); } - /** - * Sets the function to use for creating the cache key, if caching is enabled. - * - * @param cacheKeyFunction the cache key function to use - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { - this.cacheKeyFunction = cacheKeyFunction; - return this; - } - /** * Gets the (optional) cache map implementation that is used for caching, if caching is enabled. *

@@ -188,17 +196,6 @@ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { return Optional.ofNullable(cacheMap); } - /** - * Sets the cache map implementation to use for caching, if caching is enabled. - * - * @param cacheMap the cache map instance - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setCacheMap(CacheMap cacheMap) { - this.cacheMap = cacheMap; - return this; - } /** * Gets the maximum number of keys that will be presented to the {@link BatchLoader} function @@ -210,19 +207,6 @@ public int maxBatchSize() { return maxBatchSize; } - /** - * Sets the maximum number of keys that will be presented to the {@link BatchLoader} function - * before they are split into multiple class - * - * @param maxBatchSize the maximum batch size - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { - this.maxBatchSize = maxBatchSize; - return this; - } - /** * @return the statistics collector to use with these options */ @@ -230,20 +214,6 @@ public StatisticsCollector getStatisticsCollector() { return nonNull(this.statisticsCollector.get()); } - /** - * Sets the statistics collector supplier that will be used with these data loader options. Since it uses - * the supplier pattern, you can create a new statistics collector on each call or you can reuse - * a common value - * - * @param statisticsCollector the statistics collector to use - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setStatisticsCollector(Supplier statisticsCollector) { - this.statisticsCollector = nonNull(statisticsCollector); - return this; - } - /** * @return the batch environment provider that will be used to give context to batch load functions */ @@ -251,18 +221,6 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { return environmentProvider; } - /** - * Sets the batch loader environment provider that will be used to give context to batch load functions - * - * @param contextProvider the batch loader context provider - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { - this.environmentProvider = nonNull(contextProvider); - return this; - } - /** * Gets the (optional) cache store implementation that is used for value caching, if caching is enabled. *

@@ -274,17 +232,6 @@ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvide return Optional.ofNullable(valueCache); } - /** - * Sets the value cache implementation to use for caching values, if caching is enabled. - * - * @param valueCache the value cache instance - * - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setValueCache(ValueCache valueCache) { - this.valueCache = valueCache; - return this; - } /** * @return the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used @@ -294,14 +241,191 @@ public ValueCacheOptions getValueCacheOptions() { } /** - * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used - * - * @param valueCacheOptions the value cache options - * - * @return the data loader options for fluent coding + * @return the {@link BatchLoaderScheduler} to use, which can be null + */ + public BatchLoaderScheduler getBatchLoaderScheduler() { + return batchLoaderScheduler; + } + + /** + * @return the {@link DataLoaderInstrumentation} to use */ - public DataLoaderOptions setValueCacheOptions(ValueCacheOptions valueCacheOptions) { - this.valueCacheOptions = Assertions.nonNull(valueCacheOptions); - return this; + public DataLoaderInstrumentation getInstrumentation() { + return instrumentation; + } + + public static class Builder { + private boolean batchingEnabled; + private boolean cachingEnabled; + private boolean cachingExceptionsEnabled; + private CacheKey cacheKeyFunction; + private CacheMap cacheMap; + private ValueCache valueCache; + private int maxBatchSize; + private Supplier statisticsCollector; + private BatchLoaderContextProvider environmentProvider; + private ValueCacheOptions valueCacheOptions; + private BatchLoaderScheduler batchLoaderScheduler; + private DataLoaderInstrumentation instrumentation; + + public Builder() { + this(new DataLoaderOptions()); // use the defaults of the DataLoaderOptions for this builder + } + + Builder(DataLoaderOptions other) { + this.batchingEnabled = other.batchingEnabled; + this.cachingEnabled = other.cachingEnabled; + this.cachingExceptionsEnabled = other.cachingExceptionsEnabled; + this.cacheKeyFunction = other.cacheKeyFunction; + this.cacheMap = other.cacheMap; + this.valueCache = other.valueCache; + this.maxBatchSize = other.maxBatchSize; + this.statisticsCollector = other.statisticsCollector; + this.environmentProvider = other.environmentProvider; + this.valueCacheOptions = other.valueCacheOptions; + this.batchLoaderScheduler = other.batchLoaderScheduler; + this.instrumentation = other.instrumentation; + } + + /** + * Sets the option that determines whether batch loading is enabled. + * + * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise + * @return this builder for fluent coding + */ + public Builder setBatchingEnabled(boolean batchingEnabled) { + this.batchingEnabled = batchingEnabled; + return this; + } + + /** + * Sets the option that determines whether caching is enabled. + * + * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise + * @return this builder for fluent coding + */ + public Builder setCachingEnabled(boolean cachingEnabled) { + this.cachingEnabled = cachingEnabled; + return this; + } + + /** + * Sets the option that determines whether exceptional values are cache enabled. + * + * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise + * @return this builder for fluent coding + */ + public Builder setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { + this.cachingExceptionsEnabled = cachingExceptionsEnabled; + return this; + } + + /** + * Sets the function to use for creating the cache key, if caching is enabled. + * + * @param cacheKeyFunction the cache key function to use + * @return this builder for fluent coding + */ + public Builder setCacheKeyFunction(CacheKey cacheKeyFunction) { + this.cacheKeyFunction = cacheKeyFunction; + return this; + } + + /** + * Sets the cache map implementation to use for caching, if caching is enabled. + * + * @param cacheMap the cache map instance + * @return this builder for fluent coding + */ + public Builder setCacheMap(CacheMap cacheMap) { + this.cacheMap = cacheMap; + return this; + } + + /** + * Sets the value cache implementation to use for caching values, if caching is enabled. + * + * @param valueCache the value cache instance + * @return this builder for fluent coding + */ + public Builder setValueCache(ValueCache valueCache) { + this.valueCache = valueCache; + return this; + } + + /** + * Sets the maximum number of keys that will be presented to the {@link BatchLoader} function + * before they are split into multiple class + * + * @param maxBatchSize the maximum batch size + * @return this builder for fluent coding + */ + public Builder setMaxBatchSize(int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + return this; + } + + /** + * Sets the statistics collector supplier that will be used with these data loader options. Since it uses + * the supplier pattern, you can create a new statistics collector on each call, or you can reuse + * a common value + * + * @param statisticsCollector the statistics collector to use + * @return this builder for fluent coding + */ + public Builder setStatisticsCollector(Supplier statisticsCollector) { + this.statisticsCollector = statisticsCollector; + return this; + } + + /** + * Sets the batch loader environment provider that will be used to give context to batch load functions + * + * @param environmentProvider the batch loader context provider + * @return this builder for fluent coding + */ + public Builder setBatchLoaderContextProvider(BatchLoaderContextProvider environmentProvider) { + this.environmentProvider = environmentProvider; + return this; + } + + /** + * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used + * + * @param valueCacheOptions the value cache options + * @return this builder for fluent coding + */ + public Builder setValueCacheOptions(ValueCacheOptions valueCacheOptions) { + this.valueCacheOptions = valueCacheOptions; + return this; + } + + /** + * Sets in a new {@link BatchLoaderScheduler} that allows the call to a {@link BatchLoader} function to be scheduled + * to some future time. + * + * @param batchLoaderScheduler the scheduler + * @return this builder for fluent coding + */ + public Builder setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { + this.batchLoaderScheduler = batchLoaderScheduler; + return this; + } + + /** + * Sets in a new {@link DataLoaderInstrumentation} + * + * @param instrumentation the new {@link DataLoaderInstrumentation} + * @return this builder for fluent coding + */ + public Builder setInstrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = nonNull(instrumentation); + return this; + } + + public DataLoaderOptions build() { + return new DataLoaderOptions(this); + } + } } diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 9b19c29..0988697 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -1,7 +1,12 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.dataloader.instrumentation.ChainedDataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.stats.Statistics; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; @@ -13,54 +18,191 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import static org.dataloader.impl.Assertions.assertState; + /** - * This allows data loaders to be registered together into a single place so + * This allows data loaders to be registered together into a single place, so * they can be dispatched as one. It also allows you to retrieve data loaders by - * name from a central place + * name from a central place. + *

+ * Notes on {@link DataLoaderInstrumentation} : A {@link DataLoaderRegistry} can have an instrumentation + * associated with it. As each {@link DataLoader} is added to the registry, the {@link DataLoaderInstrumentation} + * of the registry is applied to that {@link DataLoader}. + *

+ * The {@link DataLoader} is changed and hence the object in the registry is not the + * same one as was originally registered. So you MUST get access to the {@link DataLoader} via {@link DataLoaderRegistry#getDataLoader(String)} methods + * and not use the original {@link DataLoader} object. + *

+ * If the {@link DataLoader} has no {@link DataLoaderInstrumentation} then the registry one is added to it. If it does have one already + * then a {@link ChainedDataLoaderInstrumentation} is created with the registry {@link DataLoaderInstrumentation} in it first and then any other + * {@link DataLoaderInstrumentation}s added after that. If the registry {@link DataLoaderInstrumentation} instance and {@link DataLoader} {@link DataLoaderInstrumentation} instance + * are the same object, then nothing is changed, since the same instrumentation code is being run. */ @PublicApi +@NullMarked public class DataLoaderRegistry { - protected final Map> dataLoaders = new ConcurrentHashMap<>(); + protected final Map> dataLoaders; + protected final @Nullable DataLoaderInstrumentation instrumentation; + public DataLoaderRegistry() { + this(new ConcurrentHashMap<>(), null); } private DataLoaderRegistry(Builder builder) { - this.dataLoaders.putAll(builder.dataLoaders); + this(builder.dataLoaders, builder.instrumentation); } + protected DataLoaderRegistry(Map> dataLoaders, @Nullable DataLoaderInstrumentation instrumentation) { + this.dataLoaders = instrumentDLs(dataLoaders, instrumentation); + this.instrumentation = instrumentation; + } + + private Map> instrumentDLs(Map> incomingDataLoaders, @Nullable DataLoaderInstrumentation registryInstrumentation) { + Map> dataLoaders = new ConcurrentHashMap<>(incomingDataLoaders); + if (registryInstrumentation != null) { + dataLoaders.replaceAll((k, existingDL) -> nameAndInstrumentDL(k, registryInstrumentation, existingDL)); + } + return dataLoaders; + } /** - * This will register a new dataloader + * Can be called to tweak a {@link DataLoader} so that it has the registry {@link DataLoaderInstrumentation} added as the first one. + * + * @param key the key used to register the data loader + * @param registryInstrumentation the common registry {@link DataLoaderInstrumentation} + * @param existingDL the existing data loader + * @return a new {@link DataLoader} or the same one if there is nothing to change + */ + private static DataLoader nameAndInstrumentDL(String key, @Nullable DataLoaderInstrumentation registryInstrumentation, DataLoader existingDL) { + existingDL = checkAndSetName(key, existingDL); + + if (registryInstrumentation == null) { + return existingDL; + } + DataLoaderOptions options = existingDL.getOptions(); + DataLoaderInstrumentation existingInstrumentation = options.getInstrumentation(); + // if they have any instrumentations then add to it + if (existingInstrumentation != null) { + if (existingInstrumentation == registryInstrumentation) { + // nothing to change + return existingDL; + } + if (existingInstrumentation == DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION) { + // replace it with the registry one + return mkInstrumentedDataLoader(existingDL, options, registryInstrumentation); + } + if (existingInstrumentation instanceof ChainedDataLoaderInstrumentation) { + // avoids calling a chained inside a chained + DataLoaderInstrumentation newInstrumentation = ((ChainedDataLoaderInstrumentation) existingInstrumentation).prepend(registryInstrumentation); + return mkInstrumentedDataLoader(existingDL, options, newInstrumentation); + } else { + DataLoaderInstrumentation newInstrumentation = new ChainedDataLoaderInstrumentation().add(registryInstrumentation).add(existingInstrumentation); + return mkInstrumentedDataLoader(existingDL, options, newInstrumentation); + } + } else { + return mkInstrumentedDataLoader(existingDL, options, registryInstrumentation); + } + } + + private static DataLoader checkAndSetName(String key, DataLoader dataLoader) { + if (dataLoader.getName() == null) { + return dataLoader.transform(b -> b.name(key)); + } + assertState(key.equals(dataLoader.getName()), + () -> String.format("Data loader name '%s' is not the same as registered key '%s'", dataLoader.getName(), key)); + return dataLoader; + } + + private static DataLoader mkInstrumentedDataLoader(DataLoader existingDL, DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { + return existingDL.transform(builder -> builder.options(setInInstrumentation(options, newInstrumentation))); + } + + private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { + return options.transform(optionsBuilder -> optionsBuilder.setInstrumentation(newInstrumentation)); + } + + /** + * @return the {@link DataLoaderInstrumentation} associated with this registry which can be null + */ + public @Nullable DataLoaderInstrumentation getInstrumentation() { + return instrumentation; + } + + /** + * This will register a new named dataloader. The {@link DataLoader} must be named something and + * cannot have a null name. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. + * + * @param dataLoader the named data loader to register + * @return this registry + */ + public DataLoaderRegistry register(DataLoader dataLoader) { + String name = dataLoader.getName(); + assertState(name != null, () -> "The DataLoader must have a non null name"); + dataLoaders.put(name, nameAndInstrumentDL(name, instrumentation, dataLoader)); + return this; + } + + /** + * This will register a new {@link DataLoader} + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this registry */ public DataLoaderRegistry register(String key, DataLoader dataLoader) { - dataLoaders.put(key, dataLoader); + dataLoaders.put(key, nameAndInstrumentDL(key, instrumentation, dataLoader)); return this; } + /** + * This will register a new {@link DataLoader} and then return it. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. + * + * @param key the key to put the data loader under + * @param dataLoader the data loader to register + * @return the data loader instance that was registered + */ + public DataLoader registerAndGet(String key, DataLoader dataLoader) { + dataLoaders.put(key, nameAndInstrumentDL(key, instrumentation, dataLoader)); + return getDataLoader(key); + } + /** * Computes a data loader if absent or return it if it was * already registered at that key. *

* Note: The entire method invocation is performed atomically, * so the function is applied at most once per key. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. * * @param key the key of the data loader * @param mappingFunction the function to compute a data loader * @param the type of keys * @param the type of values - * * @return a data loader */ @SuppressWarnings("unchecked") public DataLoader computeIfAbsent(final String key, final Function> mappingFunction) { - return (DataLoader) dataLoaders.computeIfAbsent(key, mappingFunction); + return (DataLoader) dataLoaders.computeIfAbsent(key, (k) -> { + DataLoader dl = mappingFunction.apply(k); + return nameAndInstrumentDL(key, instrumentation, dl); + }); } /** @@ -68,7 +210,6 @@ public DataLoader computeIfAbsent(final String key, * and return a new combined registry * * @param registry the registry to combine into this registry - * * @return a new combined registry */ public DataLoaderRegistry combine(DataLoaderRegistry registry) { @@ -97,7 +238,6 @@ public DataLoaderRegistry combine(DataLoaderRegistry registry) { * This will unregister a new dataloader * * @param key the key of the data loader to unregister - * * @return this registry */ public DataLoaderRegistry unregister(String key) { @@ -111,7 +251,6 @@ public DataLoaderRegistry unregister(String key) { * @param key the key of the data loader * @param the type of keys * @param the type of values - * * @return a data loader or null if its not present */ @SuppressWarnings("unchecked") @@ -127,7 +266,7 @@ public Set getKeys() { } /** - * This will called {@link org.dataloader.DataLoader#dispatch()} on each of the registered + * This will be called {@link org.dataloader.DataLoader#dispatch()} on each of the registered * {@link org.dataloader.DataLoader}s */ public void dispatchAll() { @@ -182,13 +321,13 @@ public static Builder newRegistry() { public static class Builder { private final Map> dataLoaders = new HashMap<>(); + private @Nullable DataLoaderInstrumentation instrumentation; /** * This will register a new dataloader * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this builder for a fluent pattern */ public Builder register(String key, DataLoader dataLoader) { @@ -197,11 +336,10 @@ public Builder register(String key, DataLoader dataLoader) { } /** - * This will combine together the data loaders in this builder with the ones + * This will combine the data loaders in this builder with the ones * from a previous {@link DataLoaderRegistry} * * @param otherRegistry the previous {@link DataLoaderRegistry} - * * @return this builder for a fluent pattern */ public Builder registerAll(DataLoaderRegistry otherRegistry) { @@ -209,6 +347,11 @@ public Builder registerAll(DataLoaderRegistry otherRegistry) { return this; } + public Builder instrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = instrumentation; + return this; + } + /** * @return the newly built {@link DataLoaderRegistry} */ diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java new file mode 100644 index 0000000..7cffced --- /dev/null +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -0,0 +1,188 @@ +package org.dataloader; + +import org.dataloader.annotations.PublicApi; +import org.dataloader.stats.Statistics; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * This delegating {@link DataLoader} makes it easier to create wrappers of {@link DataLoader}s in case you want to change how + * values are returned for example. + *

+ * The most common way would be to make a new {@link DelegatingDataLoader} subclass that overloads the {@link DelegatingDataLoader#load(Object, Object)} + * method. + *

+ * For example the following allows you to change the returned value in some way : + *

{@code
+ * DataLoader rawLoader = createDataLoader();
+ * DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) {
+ *    public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) {
+ *       CompletableFuture cf = super.load(key, keyContext);
+ *       return cf.thenApply(v -> "|" + v + "|");
+ *    }
+ * };
+ * }
+ * + * @param type parameter indicating the type of the data load keys + * @param type parameter indicating the type of the data that is returned + */ +@PublicApi +@NullMarked +public class DelegatingDataLoader extends DataLoader { + + protected final DataLoader delegate; + + /** + * This can be called to unwrap a given {@link DataLoader} such that if it's a {@link DelegatingDataLoader} the underlying + * {@link DataLoader} is returned otherwise it's just passed in data loader + * + * @param dataLoader the dataLoader to unwrap + * @param type parameter indicating the type of the data load keys + * @param type parameter indicating the type of the data that is returned + * @return the delegate dataLoader OR just this current one if it's not wrapped + */ + public static DataLoader unwrap(DataLoader dataLoader) { + if (dataLoader instanceof DelegatingDataLoader) { + return ((DelegatingDataLoader) dataLoader).getDelegate(); + } + return dataLoader; + } + + public DelegatingDataLoader(DataLoader delegate) { + super(delegate.getName(), delegate.getBatchLoadFunction(), delegate.getOptions()); + this.delegate = delegate; + } + + public DataLoader getDelegate() { + return delegate; + } + + /** + * The {@link DataLoader#load(Object)} and {@link DataLoader#loadMany(List)} type methods all call back + * to the {@link DataLoader#load(Object, Object)} and hence we don't override them. + * + * @param key the key to load + * @param keyContext a context object that is specific to this key + * @return the future of the value + */ + @Override + public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { + return delegate.load(key, keyContext); + } + + @Override + public DataLoader transform(Consumer> builderConsumer) { + return delegate.transform(builderConsumer); + } + + @Override + public Instant getLastDispatchTime() { + return delegate.getLastDispatchTime(); + } + + @Override + public Duration getTimeSinceDispatch() { + return delegate.getTimeSinceDispatch(); + } + + @Override + public Optional> getIfPresent(K key) { + return delegate.getIfPresent(key); + } + + @Override + public Optional> getIfCompleted(K key) { + return delegate.getIfCompleted(key); + } + + @Override + public CompletableFuture> dispatch() { + return delegate.dispatch(); + } + + @Override + public DispatchResult dispatchWithCounts() { + return delegate.dispatchWithCounts(); + } + + @Override + public List dispatchAndJoin() { + return delegate.dispatchAndJoin(); + } + + @Override + public int dispatchDepth() { + return delegate.dispatchDepth(); + } + + @Override + public Object getCacheKey(K key) { + return delegate.getCacheKey(key); + } + + @Override + public Statistics getStatistics() { + return delegate.getStatistics(); + } + + @Override + public CacheMap getCacheMap() { + return delegate.getCacheMap(); + } + + @Override + public ValueCache getValueCache() { + return delegate.getValueCache(); + } + + @Override + public DataLoader clear(K key) { + delegate.clear(key); + return this; + } + + @Override + public DataLoader clear(K key, BiConsumer handler) { + delegate.clear(key, handler); + return this; + } + + @Override + public DataLoader clearAll() { + delegate.clearAll(); + return this; + } + + @Override + public DataLoader clearAll(BiConsumer handler) { + delegate.clearAll(handler); + return this; + } + + @Override + public DataLoader prime(K key, V value) { + delegate.prime(key, value); + return this; + } + + @Override + public DataLoader prime(K key, Exception error) { + delegate.prime(key, error); + return this; + } + + @Override + public DataLoader prime(K key, CompletableFuture value) { + delegate.prime(key, value); + return this; + } +} diff --git a/src/main/java/org/dataloader/DispatchResult.java b/src/main/java/org/dataloader/DispatchResult.java index 97711da..7305c78 100644 --- a/src/main/java/org/dataloader/DispatchResult.java +++ b/src/main/java/org/dataloader/DispatchResult.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.jspecify.annotations.NullMarked; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -12,6 +13,7 @@ * @param for two */ @PublicApi +@NullMarked public class DispatchResult { private final CompletableFuture> futureList; private final int keysCount; diff --git a/src/main/java/org/dataloader/MappedBatchLoader.java b/src/main/java/org/dataloader/MappedBatchLoader.java index 4b489fa..1ad4c79 100644 --- a/src/main/java/org/dataloader/MappedBatchLoader.java +++ b/src/main/java/org/dataloader/MappedBatchLoader.java @@ -16,13 +16,15 @@ package org.dataloader; -import java.util.List; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; /** - * A function that is invoked for batch loading a map of of data values indicated by the provided set of keys. The + * A function that is invoked for batch loading a map of data values indicated by the provided set of keys. The * function returns a promise of a map of results of individual load requests. *

* There are a few constraints that must be upheld: @@ -55,6 +57,8 @@ * @param type parameter indicating the type of values returned * */ +@PublicSpi +@NullMarked public interface MappedBatchLoader { /** diff --git a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java index 6e3a2f0..9559260 100644 --- a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java @@ -16,7 +16,9 @@ package org.dataloader; -import java.util.List; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; @@ -29,6 +31,8 @@ * See {@link MappedBatchLoader} for more details on the design invariants that you must implement in order to * use this interface. */ +@PublicSpi +@NullMarked public interface MappedBatchLoaderWithContext { /** * Called to batch load the provided keys and return a promise to a map of values. diff --git a/src/main/java/org/dataloader/MappedBatchPublisher.java b/src/main/java/org/dataloader/MappedBatchPublisher.java new file mode 100644 index 0000000..493401f --- /dev/null +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -0,0 +1,34 @@ +package org.dataloader; + +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; +import org.reactivestreams.Subscriber; + +import java.util.Map; +import java.util.Set; + +/** + * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. + *

+ * The function must call the provided {@link Subscriber} to process the key/value pairs it has retrieved to allow + * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available + * (rather than when all values have been retrieved). + * + * @param type parameter indicating the type of keys to use for data load requests. + * @param type parameter indicating the type of values returned + * @see MappedBatchLoader for the non-reactive version + */ +@PublicSpi +@NullMarked +public interface MappedBatchPublisher { + /** + * Called to batch the provided keys into a stream of map entries of keys and values. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + */ + void load(Set keys, Subscriber> subscriber); +} diff --git a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java new file mode 100644 index 0000000..7b862ca --- /dev/null +++ b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java @@ -0,0 +1,36 @@ +package org.dataloader; + +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; +import org.reactivestreams.Subscriber; + +import java.util.List; +import java.util.Map; + +/** + * This form of {@link MappedBatchPublisher} is given a {@link org.dataloader.BatchLoaderEnvironment} object + * that encapsulates the calling context. A typical use case is passing in security credentials or database details + * for example. + *

+ * See {@link MappedBatchPublisher} for more details on the design invariants that you must implement in order to + * use this interface. + */ +@PublicSpi +@NullMarked +public interface MappedBatchPublisherWithContext { + + /** + * Called to batch the provided keys into a stream of map entries of keys and values. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * This is given an environment object to that maybe be useful during the call. A typical use case + * is passing in security credentials or database details for example. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + * @param environment an environment object that can help with the call + */ + void load(List keys, Subscriber> subscriber, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/Try.java b/src/main/java/org/dataloader/Try.java index 3f9a129..cd33afd 100644 --- a/src/main/java/org/dataloader/Try.java +++ b/src/main/java/org/dataloader/Try.java @@ -15,7 +15,7 @@ /** * Try is class that allows you to hold the result of computation or the throwable it produced. * - * This class is useful in {@link org.dataloader.BatchLoader}s so you can mix a batch of calls where some of + * This class is useful in {@link org.dataloader.BatchLoader}s so you can mix a batch of calls where some * the calls succeeded and some of them failed. You would make your batch loader declaration like : * *

@@ -89,8 +89,8 @@ public static  Try failed(Throwable throwable) {
     }
 
     /**
-     * This returns a Try that has always failed with an consistent exception.  Use this when
-     * yiu dont care about the exception but only that the Try failed.
+     * This returns a Try that has always failed with a consistent exception.  Use this when
+     * you don't care about the exception but only that the Try failed.
      *
      * @param  the type of value
      *
@@ -167,7 +167,7 @@ public V get() {
      */
     public Throwable getThrowable() {
         if (isSuccess()) {
-            throw new UnsupportedOperationException("You have called Try.getThrowable() with a failed Try", throwable);
+            throw new UnsupportedOperationException("You have called Try.getThrowable() with a successful Try");
         }
         return throwable;
     }
diff --git a/src/main/java/org/dataloader/ValueCache.java b/src/main/java/org/dataloader/ValueCache.java
index 551dc5d..80c8402 100644
--- a/src/main/java/org/dataloader/ValueCache.java
+++ b/src/main/java/org/dataloader/ValueCache.java
@@ -3,6 +3,7 @@
 import org.dataloader.annotations.PublicSpi;
 import org.dataloader.impl.CompletableFutureKit;
 import org.dataloader.impl.NoOpValueCache;
+import org.jspecify.annotations.NullMarked;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -15,11 +16,11 @@
  * 

* It differs from {@link CacheMap} which is in fact a cache of promised values aka {@link CompletableFuture}<V>'s. *

- * {@link ValueCache} is more suited to be a wrapper of a long-lived or externallly cached values. {@link CompletableFuture}s cant + * {@link ValueCache} is more suited to be a wrapper of a long-lived or externally cached values. {@link CompletableFuture}s can't * be easily placed in an external cache outside the JVM say, hence the need for the {@link ValueCache}. *

* {@link DataLoader}s use a two stage cache strategy if caching is enabled. If the {@link CacheMap} already has the promise to a value - * that is used. If not then the {@link ValueCache} is asked for a value, if it has one then that is returned (and cached as a promise in the {@link CacheMap}. + * that is used. If not then the {@link ValueCache} is asked for a value, if it has one then that is returned (and cached as a promise in the {@link CacheMap}). *

* If there is no value then the key is queued and loaded via the {@link BatchLoader} calls. The returned values will then be stored in * the {@link ValueCache} and the promises to those values are also stored in the {@link CacheMap}. @@ -29,7 +30,7 @@ * out of the box. *

* The API signature uses {@link CompletableFuture}s because the backing implementation MAY be a remote external cache - * and hence exceptions may happen in retrieving values and they may take time to complete. + * and hence exceptions may happen in retrieving values, and they may take time to complete. * * @param the type of cache keys * @param the type of cache values @@ -38,6 +39,7 @@ * @author Brad Baker */ @PublicSpi +@NullMarked public interface ValueCache { /** @@ -67,8 +69,8 @@ static ValueCache defaultValueCache() { CompletableFuture get(K key); /** - * Gets the specified keys from the value cache, in a batch call. If your underlying cache cant do batch caching retrieval - * then do not implement this method and it will delegate back to {@link #get(Object)} for you + * Gets the specified keys from the value cache, in a batch call. If your underlying cache cannot do batch caching retrieval + * then do not implement this method, and it will delegate back to {@link #get(Object)} for you *

* Each item in the returned list of values is a {@link Try}. If the key could not be found then a failed Try just be returned otherwise * a successful Try contain the cached value is returned. @@ -104,8 +106,8 @@ default CompletableFuture>> getValues(List keys) throws ValueCach CompletableFuture set(K key, V value); /** - * Stores the value with the specified keys, or updates it if the keys if they already exist. If your underlying cache cant do batch caching setting - * then do not implement this method and it will delegate back to {@link #set(Object, Object)} for you + * Stores the value with the specified keys, or updates it if the keys if they already exist. If your underlying cache can't do batch caching setting + * then do not implement this method, and it will delegate back to {@link #set(Object, Object)} for you * * @param keys the keys to store * @param values the values to store @@ -115,7 +117,7 @@ default CompletableFuture>> getValues(List keys) throws ValueCach * @throws ValueCachingNotSupported if this cache wants to short-circuit this method completely */ default CompletableFuture> setValues(List keys, List values) throws ValueCachingNotSupported { - List> cacheSets = new ArrayList<>(); + List> cacheSets = new ArrayList<>(keys.size()); for (int i = 0; i < keys.size(); i++) { K k = keys.get(i); V v = values.get(i); @@ -158,4 +160,4 @@ public Throwable fillInStackTrace() { return this; } } -} \ No newline at end of file +} diff --git a/src/main/java/org/dataloader/ValueCacheOptions.java b/src/main/java/org/dataloader/ValueCacheOptions.java index 1a0c1a1..b681dda 100644 --- a/src/main/java/org/dataloader/ValueCacheOptions.java +++ b/src/main/java/org/dataloader/ValueCacheOptions.java @@ -1,10 +1,15 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + /** * Options that control how the {@link ValueCache} is used by {@link DataLoader} * * @author Brad Baker */ +@PublicSpi +@NullMarked public class ValueCacheOptions { private final boolean completeValueAfterCacheSet; @@ -22,7 +27,7 @@ public static ValueCacheOptions newOptions() { /** * This controls whether the {@link DataLoader} will wait for the {@link ValueCache#set(Object, Object)} call - * to complete before it completes the returned value. By default this is false and hence + * to complete before it completes the returned value. By default, this is false and hence * the {@link ValueCache#set(Object, Object)} call may complete some time AFTER the data loader * value has been returned. * diff --git a/src/main/java/org/dataloader/annotations/ExperimentalApi.java b/src/main/java/org/dataloader/annotations/ExperimentalApi.java index 6be889e..782998e 100644 --- a/src/main/java/org/dataloader/annotations/ExperimentalApi.java +++ b/src/main/java/org/dataloader/annotations/ExperimentalApi.java @@ -14,7 +14,7 @@ * This represents code that the graphql-java project considers experimental API and while our intention is that it will * progress to be {@link PublicApi}, its existence, signature of behavior may change between releases. * - * In general unnecessary changes will be avoided but you should not depend on experimental classes being stable + * In general unnecessary changes will be avoided, but you should not depend on experimental classes being stable */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, TYPE, FIELD}) diff --git a/src/main/java/org/dataloader/annotations/GuardedBy.java b/src/main/java/org/dataloader/annotations/GuardedBy.java index c26b2ef..85c5765 100644 --- a/src/main/java/org/dataloader/annotations/GuardedBy.java +++ b/src/main/java/org/dataloader/annotations/GuardedBy.java @@ -15,7 +15,7 @@ public @interface GuardedBy { /** - * The lock that should be held. + * @return The lock that should be held. */ String value(); } diff --git a/src/main/java/org/dataloader/annotations/Internal.java b/src/main/java/org/dataloader/annotations/Internal.java index 4ad04cd..51cfef2 100644 --- a/src/main/java/org/dataloader/annotations/Internal.java +++ b/src/main/java/org/dataloader/annotations/Internal.java @@ -13,7 +13,7 @@ * This represents code that the java-dataloader project considers internal code that MAY not be stable within * major releases. * - * In general unnecessary changes will be avoided but you should not depend on internal classes being stable + * In general unnecessary changes will be avoided, but you should not depend on internal classes being stable */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, TYPE, FIELD}) diff --git a/src/main/java/org/dataloader/annotations/PublicSpi.java b/src/main/java/org/dataloader/annotations/PublicSpi.java index 5f385b7..7384fa9 100644 --- a/src/main/java/org/dataloader/annotations/PublicSpi.java +++ b/src/main/java/org/dataloader/annotations/PublicSpi.java @@ -15,7 +15,7 @@ * * The guarantee is for callers of code with this annotation as well as derivations that inherit / implement this code. * - * New methods will not be added (without using default methods say) that would nominally breaks SPI implementations + * New methods will not be added (without using default methods say) that would nominally break SPI implementations * within a major release. */ @Retention(RetentionPolicy.RUNTIME) diff --git a/src/main/java/org/dataloader/annotations/VisibleForTesting.java b/src/main/java/org/dataloader/annotations/VisibleForTesting.java index 99f97f0..a391113 100644 --- a/src/main/java/org/dataloader/annotations/VisibleForTesting.java +++ b/src/main/java/org/dataloader/annotations/VisibleForTesting.java @@ -9,7 +9,7 @@ import static java.lang.annotation.ElementType.METHOD; /** - * Marks fields, methods etc as more visible than actually needed for testing purposes. + * Marks fields, methods etc. as more visible than actually needed for testing purposes. */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, FIELD}) diff --git a/src/main/java/org/dataloader/impl/CompletableFutureKit.java b/src/main/java/org/dataloader/impl/CompletableFutureKit.java index 2b94d10..ebc35ec 100644 --- a/src/main/java/org/dataloader/impl/CompletableFutureKit.java +++ b/src/main/java/org/dataloader/impl/CompletableFutureKit.java @@ -3,8 +3,10 @@ import org.dataloader.annotations.Internal; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -48,10 +50,21 @@ public static boolean failed(CompletableFuture future) { } public static CompletableFuture> allOf(List> cfs) { - return CompletableFuture.allOf(cfs.toArray(new CompletableFuture[0])) + return CompletableFuture.allOf(cfs.toArray(CompletableFuture[]::new)) .thenApply(v -> cfs.stream() .map(CompletableFuture::join) .collect(toList()) ); } + + public static CompletableFuture> allOf(Map> cfs) { + return CompletableFuture.allOf(cfs.values().toArray(CompletableFuture[]::new)) + .thenApply(v -> cfs.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + task -> task.getValue().join()) + ) + ); + } } diff --git a/src/main/java/org/dataloader/impl/PromisedValues.java b/src/main/java/org/dataloader/impl/PromisedValues.java index 89ae8de..ab75044 100644 --- a/src/main/java/org/dataloader/impl/PromisedValues.java +++ b/src/main/java/org/dataloader/impl/PromisedValues.java @@ -12,11 +12,11 @@ import static java.util.Arrays.asList; /** - * This allows multiple {@link CompletionStage}s to be combined together and completed + * This allows multiple {@link CompletionStage}s to be combined and completed * as one and should something go wrong, instead of throwing {@link CompletionException}s it captures the cause and returns null for that - * data value, other wise it allows you to access them as a list of values. + * data value, otherwise it allows you to access them as a list of values. *

- * This class really encapsulate a list of promised values. It is considered finished when all of the underlying futures + * This class really encapsulate a list of promised values. It is considered finished when all the underlying futures * are finished. *

* You can get that list of values via {@link #toList()}. You can also compose a {@link CompletableFuture} of that @@ -28,7 +28,7 @@ public interface PromisedValues { /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -43,7 +43,7 @@ static PromisedValues allOf(List> cfs) { } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -59,7 +59,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2) } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -77,7 +77,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2, /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -96,7 +96,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2, /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -111,7 +111,7 @@ static PromisedValues allPromisedValues(List> cfs) { } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -127,7 +127,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -144,7 +144,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -177,7 +177,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa boolean succeeded(); /** - * @return true if any of the the futures completed unsuccessfully + * @return true if any of the futures completed unsuccessfully */ boolean failed(); @@ -220,7 +220,6 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa * * @return the value of the future */ - @SuppressWarnings("unchecked") T get(int index); /** diff --git a/src/main/java/org/dataloader/impl/PromisedValuesImpl.java b/src/main/java/org/dataloader/impl/PromisedValuesImpl.java index 2ba592b..ddaba81 100644 --- a/src/main/java/org/dataloader/impl/PromisedValuesImpl.java +++ b/src/main/java/org/dataloader/impl/PromisedValuesImpl.java @@ -25,7 +25,7 @@ public class PromisedValuesImpl implements PromisedValues { private PromisedValuesImpl(List> cs) { this.futures = nonNull(cs); this.cause = new AtomicReference<>(); - CompletableFuture[] futuresArray = cs.stream().map(CompletionStage::toCompletableFuture).toArray(CompletableFuture[]::new); + CompletableFuture[] futuresArray = cs.stream().map(CompletionStage::toCompletableFuture).toArray(CompletableFuture[]::new); this.controller = CompletableFuture.allOf(futuresArray).handle((result, throwable) -> { setCause(throwable); return null; diff --git a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java new file mode 100644 index 0000000..bf8a40c --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java @@ -0,0 +1,124 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; +import org.dataloader.annotations.PublicApi; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This {@link DataLoaderInstrumentation} can chain together multiple instrumentations and have them all called in + * the order of the provided list. + */ +@PublicApi +public class ChainedDataLoaderInstrumentation implements DataLoaderInstrumentation { + private final List instrumentations; + + public ChainedDataLoaderInstrumentation() { + instrumentations = List.of(); + } + + public ChainedDataLoaderInstrumentation(List instrumentations) { + this.instrumentations = List.copyOf(instrumentations); + } + + public List getInstrumentations() { + return instrumentations; + } + + /** + * Adds a new {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentation the one to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation add(DataLoaderInstrumentation instrumentation) { + ArrayList list = new ArrayList<>(this.instrumentations); + list.add(instrumentation); + return new ChainedDataLoaderInstrumentation(list); + } + + /** + * Prepends a new {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentation the one to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation prepend(DataLoaderInstrumentation instrumentation) { + ArrayList list = new ArrayList<>(); + list.add(instrumentation); + list.addAll(this.instrumentations); + return new ChainedDataLoaderInstrumentation(list); + } + + /** + * Adds a collection of {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentations the new ones to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation addAll(Collection instrumentations) { + ArrayList list = new ArrayList<>(this.instrumentations); + list.addAll(instrumentations); + return new ChainedDataLoaderInstrumentation(list); + } + + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + return chainedCtx(it -> it.beginLoad(dataLoader, key, loadContext)); + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + return chainedCtx(it -> it.beginDispatch(dataLoader)); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return chainedCtx(it -> it.beginBatchLoader(dataLoader, keys, environment)); + } + + private DataLoaderInstrumentationContext chainedCtx(Function> mapper) { + // if we have zero or 1 instrumentations (and 1 is the most common), then we can avoid an object allocation + // of the ChainedInstrumentationContext since it won't be needed + if (instrumentations.isEmpty()) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + if (instrumentations.size() == 1) { + return mapper.apply(instrumentations.get(0)); + } + return new ChainedInstrumentationContext<>(dropNullContexts(mapper)); + } + + private List> dropNullContexts(Function> mapper) { + return instrumentations.stream() + .map(mapper) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private static class ChainedInstrumentationContext implements DataLoaderInstrumentationContext { + private final List> contexts; + + public ChainedInstrumentationContext(List> contexts) { + this.contexts = contexts; + } + + @Override + public void onDispatched() { + contexts.forEach(DataLoaderInstrumentationContext::onDispatched); + } + + @Override + public void onCompleted(T result, Throwable t) { + contexts.forEach(it -> it.onCompleted(result, t)); + } + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java new file mode 100644 index 0000000..bbdba87 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java @@ -0,0 +1,53 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; +import org.dataloader.annotations.PublicSpi; + +import java.util.List; + +/** + * This interface is called when certain actions happen inside a data loader + */ +@PublicSpi +public interface DataLoaderInstrumentation { + /** + * This call back is done just before the {@link DataLoader#load(Object)} methods are invoked, + * and it completes when the load promise is completed. If the value is a cached {@link java.util.concurrent.CompletableFuture} + * then it might return almost immediately, otherwise it will return + * when the batch load function is invoked and values get returned + * + * @param dataLoader the {@link DataLoader} in question + * @param key the key used during the {@link DataLoader#load(Object)} call + * @param loadContext the load context used during the {@link DataLoader#load(Object, Object)} call + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + return null; + } + + /** + * This call back is done just before the {@link DataLoader#dispatch()} is invoked, + * and it completes when the dispatch call promise is done. + * + * @param dataLoader the {@link DataLoader} in question + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + return null; + } + + /** + * This call back is done just before the `batch loader` of a {@link DataLoader} is invoked. Remember a batch loader + * could be called multiple times during a dispatch event (because of max batch sizes) + * + * @param dataLoader the {@link DataLoader} in question + * @param keys the set of keys being fetched + * @param environment the {@link BatchLoaderEnvironment} + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return null; + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java new file mode 100644 index 0000000..88b08ef --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java @@ -0,0 +1,33 @@ +package org.dataloader.instrumentation; + +import org.dataloader.annotations.PublicSpi; + +import java.util.concurrent.CompletableFuture; + +/** + * When a {@link DataLoaderInstrumentation}.'beginXXX()' method is called then it must return a {@link DataLoaderInstrumentationContext} + * that will be invoked when the step is first dispatched and then when it completes. Sometimes this is effectively the same time + * whereas at other times it's when an asynchronous {@link CompletableFuture} completes. + *

+ * This pattern of construction of an object then call back is intended to allow "timers" to be created that can instrument what has + * just happened or "loggers" to be called to record what has happened. + */ +@PublicSpi +public interface DataLoaderInstrumentationContext { + /** + * This is invoked when the instrumentation step is initially dispatched. Note this is NOT + * the same time as the {@link DataLoaderInstrumentation}`beginXXX()` starts, but rather after all the inner + * work has been done. + */ + default void onDispatched() { + } + + /** + * This is invoked when the instrumentation step is fully completed. + * + * @param result the result of the step (which may be null) + * @param t this exception will be non-null if an exception was thrown during the step + */ + default void onCompleted(T result, Throwable t) { + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java new file mode 100644 index 0000000..9e60060 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java @@ -0,0 +1,74 @@ +package org.dataloader.instrumentation; + +import org.dataloader.annotations.PublicApi; + +import java.util.function.BiConsumer; + +@PublicApi +public class DataLoaderInstrumentationHelper { + + @SuppressWarnings("RedundantMethodOverride") + private static final DataLoaderInstrumentationContext NOOP_CTX = new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + } + + @Override + public void onCompleted(Object result, Throwable t) { + } + }; + + /** + * Returns a noop {@link DataLoaderInstrumentationContext} of the right type + * + * @param for two + * @return a noop context + */ + public static DataLoaderInstrumentationContext noOpCtx() { + //noinspection unchecked + return (DataLoaderInstrumentationContext) NOOP_CTX; + } + + /** + * A well known noop {@link DataLoaderInstrumentation} + */ + public static final DataLoaderInstrumentation NOOP_INSTRUMENTATION = new DataLoaderInstrumentation() { + }; + + /** + * Allows for the more fluent away to return an instrumentation context that runs the specified + * code on instrumentation step dispatch. + * + * @param codeToRun the code to run on dispatch + * @param the generic type + * @return an instrumentation context + */ + public static DataLoaderInstrumentationContext whenDispatched(Runnable codeToRun) { + return new SimpleDataLoaderInstrumentationContext<>(codeToRun, null); + } + + /** + * Allows for the more fluent away to return an instrumentation context that runs the specified + * code on instrumentation step completion. + * + * @param codeToRun the code to run on completion + * @param the generic type + * @return an instrumentation context + */ + public static DataLoaderInstrumentationContext whenCompleted(BiConsumer codeToRun) { + return new SimpleDataLoaderInstrumentationContext<>(null, codeToRun); + } + + + /** + * Check the {@link DataLoaderInstrumentationContext} to see if its null and returns a noop if it is or else the original + * context. This is a bit of a helper method. + * + * @param ic the context in play + * @param for two + * @return a non null context + */ + public static DataLoaderInstrumentationContext ctxOrNoopCtx(DataLoaderInstrumentationContext ic) { + return ic == null ? noOpCtx() : ic; + } +} diff --git a/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java new file mode 100644 index 0000000..f629a05 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java @@ -0,0 +1,35 @@ +package org.dataloader.instrumentation; + + +import org.dataloader.annotations.Internal; + +import java.util.function.BiConsumer; + +/** + * A simple implementation of {@link DataLoaderInstrumentationContext} + */ +@Internal +class SimpleDataLoaderInstrumentationContext implements DataLoaderInstrumentationContext { + + private final BiConsumer codeToRunOnComplete; + private final Runnable codeToRunOnDispatch; + + SimpleDataLoaderInstrumentationContext(Runnable codeToRunOnDispatch, BiConsumer codeToRunOnComplete) { + this.codeToRunOnComplete = codeToRunOnComplete; + this.codeToRunOnDispatch = codeToRunOnDispatch; + } + + @Override + public void onDispatched() { + if (codeToRunOnDispatch != null) { + codeToRunOnDispatch.run(); + } + } + + @Override + public void onCompleted(T result, Throwable t) { + if (codeToRunOnComplete != null) { + codeToRunOnComplete.accept(result, t); + } + } +} diff --git a/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java new file mode 100644 index 0000000..c2f5438 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java @@ -0,0 +1,104 @@ +package org.dataloader.reactive; + +import org.dataloader.Try; +import org.dataloader.stats.context.IncrementBatchLoadExceptionCountStatisticsContext; +import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.dataloader.impl.Assertions.assertState; + +/** + * The base class for our reactive subscriber support + * + * @param for two + */ +abstract class AbstractBatchSubscriber implements Subscriber { + + final CompletableFuture> valuesFuture; + final List keys; + final List callContexts; + final List> queuedFutures; + final ReactiveSupport.HelperIntegration helperIntegration; + + List clearCacheKeys = new ArrayList<>(); + List completedValues = new ArrayList<>(); + boolean onErrorCalled = false; + boolean onCompleteCalled = false; + + AbstractBatchSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + this.valuesFuture = valuesFuture; + this.keys = keys; + this.callContexts = callContexts; + this.queuedFutures = queuedFutures; + this.helperIntegration = helperIntegration; + } + + @Override + public void onSubscribe(Subscription subscription) { + subscription.request(keys.size()); + } + + @Override + public void onNext(T v) { + assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); + assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); + } + + @Override + public void onComplete() { + assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); + onCompleteCalled = true; + } + + @Override + public void onError(Throwable throwable) { + assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); + onErrorCalled = true; + + helperIntegration.getStats().incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); + } + + /* + * A value has arrived - how do we complete the future that's associated with it in a common way + */ + void onNextValue(K key, V value, Object callContext, List> futures) { + if (value instanceof Try) { + // we allow the batch loader to return a Try so we can better represent a computation + // that might have worked or not. + //noinspection unchecked + Try tryValue = (Try) value; + if (tryValue.isSuccess()) { + futures.forEach(f -> f.complete(tryValue.get())); + } else { + helperIntegration.getStats().incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + futures.forEach(f -> f.completeExceptionally(tryValue.getThrowable())); + clearCacheKeys.add(key); + } + } else { + futures.forEach(f -> f.complete(value)); + } + } + + Throwable unwrapThrowable(Throwable ex) { + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + return ex; + } + + void possiblyClearCacheEntriesOnExceptions() { + helperIntegration.clearCacheEntriesOnExceptions(clearCacheKeys); + } +} diff --git a/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java new file mode 100644 index 0000000..d0b8110 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java @@ -0,0 +1,86 @@ +package org.dataloader.reactive; + +import org.dataloader.impl.DataLoaderAssertionException; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** + * This class can be used to subscribe to a {@link org.reactivestreams.Publisher} and then + * have the values it receives complete the data loader keys. The keys and values must be + * in index order. + *

+ * This is a reactive version of {@link org.dataloader.BatchLoader} + * + * @param the type of keys + * @param the type of values + */ +class BatchSubscriberImpl extends AbstractBatchSubscriber { + + private int idx = 0; + + BatchSubscriberImpl( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee + // correctness (at the cost of speed). + @Override + public synchronized void onNext(V value) { + super.onNext(value); + + if (idx >= keys.size()) { + // hang on they have given us more values than we asked for in keys + // we cant handle this + return; + } + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture future = queuedFutures.get(idx); + onNextValue(key, value, callContext, List.of(future)); + + completedValues.add(value); + idx++; + } + + + @Override + public synchronized void onComplete() { + super.onComplete(); + if (keys.size() != completedValues.size()) { + // we have more or less values than promised + // we will go through all the outstanding promises and mark those that + // have not finished as failed + for (CompletableFuture queuedFuture : queuedFutures) { + if (!queuedFuture.isDone()) { + queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); + } + } + } + possiblyClearCacheEntriesOnExceptions(); + valuesFuture.complete(completedValues); + } + + @Override + public synchronized void onError(Throwable ex) { + super.onError(ex); + ex = unwrapThrowable(ex); + // Set the remaining keys to the exception. + for (int i = idx; i < queuedFutures.size(); i++) { + K key = keys.get(i); + CompletableFuture future = queuedFutures.get(i); + if (!future.isDone()) { + future.completeExceptionally(ex); + // clear any cached view of this key because it failed + helperIntegration.clearCacheView(key); + } + } + valuesFuture.completeExceptionally(ex); + } +} diff --git a/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java new file mode 100644 index 0000000..d56efa0 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java @@ -0,0 +1,103 @@ +package org.dataloader.reactive; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * This class can be used to subscribe to a {@link org.reactivestreams.Publisher} and then + * have the values it receives complete the data loader keys in a map lookup fashion. + *

+ * This is a reactive version of {@link org.dataloader.MappedBatchLoader} + * + * @param the type of keys + * @param the type of values + */ +class MappedBatchSubscriberImpl extends AbstractBatchSubscriber> { + + private final Map callContextByKey; + private final Map>> queuedFuturesByKey; + private final Map completedValuesByKey = new HashMap<>(); + + + MappedBatchSubscriberImpl( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + this.callContextByKey = new HashMap<>(); + this.queuedFuturesByKey = new HashMap<>(); + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture queuedFuture = queuedFutures.get(idx); + callContextByKey.put(key, callContext); + queuedFuturesByKey.computeIfAbsent(key, k -> new ArrayList<>()).add(queuedFuture); + } + } + + + @Override + public synchronized void onNext(Map.Entry entry) { + super.onNext(entry); + K key = entry.getKey(); + V value = entry.getValue(); + + Object callContext = callContextByKey.get(key); + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + + onNextValue(key, value, callContext, futures); + + // did we have an actual key for this value - ignore it if they send us one outside the key set + if (!futures.isEmpty()) { + completedValuesByKey.put(key, value); + } + } + + @Override + public synchronized void onComplete() { + super.onComplete(); + + possiblyClearCacheEntriesOnExceptions(); + List values = new ArrayList<>(keys.size()); + for (K key : keys) { + V value = completedValuesByKey.get(key); + values.add(value); + + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + for (CompletableFuture future : futures) { + if (!future.isDone()) { + // we have a future that never came back for that key + // but the publisher is done sending in data - it must be null + // e.g. for key X when found no value + future.complete(null); + } + } + } + valuesFuture.complete(values); + } + + @Override + public synchronized void onError(Throwable ex) { + super.onError(ex); + ex = unwrapThrowable(ex); + // Complete the futures for the remaining keys with the exception. + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + List> futures = queuedFuturesByKey.get(key); + if (!completedValuesByKey.containsKey(key)) { + for (CompletableFuture future : futures) { + future.completeExceptionally(ex); + } + // clear any cached view of this key because they all failed + helperIntegration.clearCacheView(key); + } + } + valuesFuture.completeExceptionally(ex); + } +} diff --git a/src/main/java/org/dataloader/reactive/ReactiveSupport.java b/src/main/java/org/dataloader/reactive/ReactiveSupport.java new file mode 100644 index 0000000..fc03bb0 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/ReactiveSupport.java @@ -0,0 +1,45 @@ +package org.dataloader.reactive; + +import org.dataloader.stats.StatisticsCollector; +import org.reactivestreams.Subscriber; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public class ReactiveSupport { + + public static Subscriber batchSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + return new BatchSubscriberImpl<>(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + public static Subscriber> mappedBatchSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + return new MappedBatchSubscriberImpl<>(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + /** + * Just some callbacks to the data loader code to do common tasks + * + * @param for keys + */ + public interface HelperIntegration { + + StatisticsCollector getStats(); + + void clearCacheView(K key); + + void clearCacheEntriesOnExceptions(List keys); + } +} diff --git a/src/main/java/org/dataloader/registries/DispatchPredicate.java b/src/main/java/org/dataloader/registries/DispatchPredicate.java index d5bd31b..677f484 100644 --- a/src/main/java/org/dataloader/registries/DispatchPredicate.java +++ b/src/main/java/org/dataloader/registries/DispatchPredicate.java @@ -10,6 +10,16 @@ */ @FunctionalInterface public interface DispatchPredicate { + + /** + * A predicate that always returns true + */ + DispatchPredicate DISPATCH_ALWAYS = (dataLoaderKey, dataLoader) -> true; + /** + * A predicate that always returns false + */ + DispatchPredicate DISPATCH_NEVER = (dataLoaderKey, dataLoader) -> false; + /** * This predicate tests whether the data loader should be dispatched or not. * @@ -63,7 +73,7 @@ default DispatchPredicate or(DispatchPredicate other) { } /** - * This predicate will return true if the {@link DataLoader} has not be dispatched + * This predicate will return true if the {@link DataLoader} has not been dispatched * for at least the duration length of time. * * @param duration the length of time to check diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 4be317e..4f62378 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -3,10 +3,15 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderRegistry; import org.dataloader.annotations.ExperimentalApi; +import org.dataloader.impl.Assertions; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.time.Duration; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -14,32 +19,64 @@ import static org.dataloader.impl.Assertions.nonNull; /** - * This {@link DataLoaderRegistry} will use a {@link DispatchPredicate} when {@link #dispatchAll()} is called + * This {@link DataLoaderRegistry} will use {@link DispatchPredicate}s when {@link #dispatchAll()} is called * to test (for each {@link DataLoader} in the registry) if a dispatch should proceed. If the predicate returns false, then a task is scheduled * to perform that predicate dispatch again via the {@link ScheduledExecutorService}. *

+ * It;s possible to have a {@link DispatchPredicate} per dataloader as well as a default {@link DispatchPredicate} for the + * whole {@link ScheduledDataLoaderRegistry}. + *

* This will continue to loop (test false and reschedule) until such time as the predicate returns true, in which case - * no rescheduling will occur and you will need to call dispatch again to restart the process. + * no rescheduling will occur, and you will need to call dispatch again to restart the process. + *

+ * In the default mode, when {@link #tickerMode} is false, the registry will continue to loop (test false and reschedule) until such time as the predicate returns true, in which case + * no rescheduling will occur, and you will need to call dispatch again to restart the process. + *

+ * However, when {@link #tickerMode} is true, the registry will always reschedule continuously after the first ever call to {@link #dispatchAll()}. + *

+ * This will allow you to chain together {@link DataLoader} load calls like this : + *

{@code
+ *   CompletableFuture future = dataLoaderA.load("A")
+ *                                          .thenCompose(value -> dataLoaderB.load(value));
+ * }
+ *

+ * However, it may mean your batching will not be as efficient as it might be. In environments + * like graphql this might mean you are too eager in fetching. The {@link DispatchPredicate} still runs to decide if + * dispatch should happen however in ticker mode it will be continuously rescheduled. + *

+ * When {@link #tickerMode} is true, you really SHOULD close the registry say at the end of a request otherwise you will leave a job + * on the {@link ScheduledExecutorService} that is continuously dispatching. *

* If you wanted to create a ScheduledDataLoaderRegistry that started a rescheduling immediately, just create one and * call {@link #rescheduleNow()}. *

+ * By default, it uses a {@link Executors#newSingleThreadScheduledExecutor()}} to schedule the tasks. However, if you + * are creating a {@link ScheduledDataLoaderRegistry} per request you will want to look at sharing this {@link ScheduledExecutorService} + * to avoid creating a new thread per registry created. + *

* This code is currently marked as {@link ExperimentalApi} */ @ExperimentalApi +@NullMarked public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements AutoCloseable { - private final ScheduledExecutorService scheduledExecutorService; + private final Map, DispatchPredicate> dataLoaderPredicates = new ConcurrentHashMap<>(); private final DispatchPredicate dispatchPredicate; + private final ScheduledExecutorService scheduledExecutorService; + private final boolean defaultExecutorUsed; private final Duration schedule; + private final boolean tickerMode; private volatile boolean closed; private ScheduledDataLoaderRegistry(Builder builder) { - this.dataLoaders.putAll(builder.dataLoaders); - this.scheduledExecutorService = builder.scheduledExecutorService; - this.dispatchPredicate = builder.dispatchPredicate; + super(builder.dataLoaders, builder.instrumentation); + this.scheduledExecutorService = Assertions.nonNull(builder.scheduledExecutorService); + this.defaultExecutorUsed = builder.defaultExecutorUsed; this.schedule = builder.schedule; + this.tickerMode = builder.tickerMode; this.closed = false; + this.dispatchPredicate = builder.dispatchPredicate; + this.dataLoaderPredicates.putAll(builder.dataLoaderPredicates); } /** @@ -48,6 +85,16 @@ private ScheduledDataLoaderRegistry(Builder builder) { @Override public void close() { closed = true; + if (defaultExecutorUsed) { + scheduledExecutorService.shutdown(); + } + } + + /** + * @return executor being used by this registry + */ + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; } /** @@ -57,6 +104,73 @@ public Duration getScheduleDuration() { return schedule; } + /** + * @return true of the registry is in ticker mode or false otherwise + */ + public boolean isTickerMode() { + return tickerMode; + } + + /** + * This will combine all the current data loaders in this registry and all the data loaders from the specified registry + * and return a new combined registry + * + * @param registry the registry to combine into this registry + * @return a new combined registry + */ + public ScheduledDataLoaderRegistry combine(DataLoaderRegistry registry) { + Builder combinedBuilder = ScheduledDataLoaderRegistry.newScheduledRegistry() + .dispatchPredicate(this.dispatchPredicate); + combinedBuilder.registerAll(this); + combinedBuilder.registerAll(registry); + return combinedBuilder.build(); + } + + + /** + * This will unregister a new dataloader + * + * @param key the key of the data loader to unregister + * @return this registry + */ + public ScheduledDataLoaderRegistry unregister(String key) { + DataLoader dataLoader = dataLoaders.remove(key); + if (dataLoader != null) { + dataLoaderPredicates.remove(dataLoader); + } + return this; + } + + /** + * @return a map of data loaders to specific dispatch predicates + */ + public Map, DispatchPredicate> getDataLoaderPredicates() { + return new LinkedHashMap<>(dataLoaderPredicates); + } + + /** + * There is a default predicate that applies to the whole {@link ScheduledDataLoaderRegistry} + * + * @return the default dispatch predicate + */ + public DispatchPredicate getDispatchPredicate() { + return dispatchPredicate; + } + + /** + * This will register a new dataloader and dispatch predicate associated with that data loader + * + * @param key the key to put the data loader under + * @param dataLoader the data loader to register + * @param dispatchPredicate the dispatch predicate to associate with this data loader + * @return this registry + */ + public ScheduledDataLoaderRegistry register(String key, DataLoader dataLoader, DispatchPredicate dispatchPredicate) { + dataLoaders.put(key, dataLoader); + dataLoaderPredicates.put(dataLoader, dispatchPredicate); + return this; + } + @Override public void dispatchAll() { dispatchAllWithCount(); @@ -68,41 +182,57 @@ public int dispatchAllWithCount() { for (Map.Entry> entry : dataLoaders.entrySet()) { DataLoader dataLoader = entry.getValue(); String key = entry.getKey(); - if (dispatchPredicate.test(key, dataLoader)) { - sum += dataLoader.dispatchWithCounts().getKeysCount(); - } else { - reschedule(key, dataLoader); - } + sum += dispatchOrReschedule(key, dataLoader); } return sum; } + /** * This will immediately dispatch the {@link DataLoader}s in the registry - * without testing the predicate + * without testing the predicates */ public void dispatchAllImmediately() { - super.dispatchAll(); + dispatchAllWithCountImmediately(); } /** * This will immediately dispatch the {@link DataLoader}s in the registry - * without testing the predicate + * without testing the predicates * * @return total number of entries that were dispatched from registered {@link org.dataloader.DataLoader}s. */ public int dispatchAllWithCountImmediately() { - return super.dispatchAllWithCount(); + return dataLoaders.values().stream() + .mapToInt(dataLoader -> dataLoader.dispatchWithCounts().getKeysCount()) + .sum(); } + /** * This will schedule a task to check the predicate and dispatch if true right now. It will not do - * a pre check of the preodicate like {@link #dispatchAll()} would + * a pre-check of the predicate like {@link #dispatchAll()} would */ public void rescheduleNow() { dataLoaders.forEach(this::reschedule); } + /** + * If a specific {@link DispatchPredicate} is registered for this dataloader then it uses it values + * otherwise the overall registry predicate is used. + * + * @param dataLoaderKey the key in the dataloader map + * @param dataLoader the dataloader + * @return true if it should dispatch + */ + private boolean shouldDispatch(String dataLoaderKey, DataLoader dataLoader) { + DispatchPredicate dispatchPredicate = dataLoaderPredicates.get(dataLoader); + if (dispatchPredicate != null) { + return dispatchPredicate.test(dataLoaderKey, dataLoader); + } + return this.dispatchPredicate.test(dataLoaderKey, dataLoader); + } + private void reschedule(String key, DataLoader dataLoader) { if (!closed) { Runnable runThis = () -> dispatchOrReschedule(key, dataLoader); @@ -110,17 +240,21 @@ private void reschedule(String key, DataLoader dataLoader) { } } - private void dispatchOrReschedule(String key, DataLoader dataLoader) { - if (dispatchPredicate.test(key, dataLoader)) { - dataLoader.dispatch(); - } else { + private int dispatchOrReschedule(String key, DataLoader dataLoader) { + int sum = 0; + boolean shouldDispatch = shouldDispatch(key, dataLoader); + if (shouldDispatch) { + sum = dataLoader.dispatchWithCounts().getKeysCount(); + } + if (tickerMode || !shouldDispatch) { reschedule(key, dataLoader); } + return sum; } /** - * By default this will create use a {@link Executors#newSingleThreadScheduledExecutor()} - * and a schedule duration of 10 milli seconds. + * By default, this will create use a {@link Executors#newSingleThreadScheduledExecutor()} + * and a schedule duration of 10 milliseconds. * * @return A builder of {@link ScheduledDataLoaderRegistry}s */ @@ -130,11 +264,23 @@ public static Builder newScheduledRegistry() { public static class Builder { - private ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private DispatchPredicate dispatchPredicate = (key, dl) -> true; + private final Map> dataLoaders = new LinkedHashMap<>(); + private final Map, DispatchPredicate> dataLoaderPredicates = new LinkedHashMap<>(); + private DispatchPredicate dispatchPredicate = DispatchPredicate.DISPATCH_ALWAYS; + private @Nullable ScheduledExecutorService scheduledExecutorService; + private boolean defaultExecutorUsed = false; private Duration schedule = Duration.ofMillis(10); - private final Map> dataLoaders = new HashMap<>(); + private boolean tickerMode = false; + private @Nullable DataLoaderInstrumentation instrumentation; + + /** + * If you provide a {@link ScheduledExecutorService} then it will NOT be shutdown when + * {@link ScheduledDataLoaderRegistry#close()} is called. This is left to the code that made this setup code + * + * @param executorService the executor service to run the ticker on + * @return this builder for a fluent pattern + */ public Builder scheduledExecutorService(ScheduledExecutorService executorService) { this.scheduledExecutorService = nonNull(executorService); return this; @@ -145,17 +291,11 @@ public Builder schedule(Duration schedule) { return this; } - public Builder dispatchPredicate(DispatchPredicate dispatchPredicate) { - this.dispatchPredicate = nonNull(dispatchPredicate); - return this; - } - /** * This will register a new dataloader * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this builder for a fluent pattern */ public Builder register(String key, DataLoader dataLoader) { @@ -163,16 +303,64 @@ public Builder register(String key, DataLoader dataLoader) { return this; } + + /** + * This will register a new dataloader with a specific {@link DispatchPredicate} + * + * @param key the key to put the data loader under + * @param dataLoader the data loader to register + * @param dispatchPredicate the dispatch predicate + * @return this builder for a fluent pattern + */ + public Builder register(String key, DataLoader dataLoader, DispatchPredicate dispatchPredicate) { + register(key, dataLoader); + dataLoaderPredicates.put(dataLoader, dispatchPredicate); + return this; + } + /** - * This will combine together the data loaders in this builder with the ones + * This will combine the data loaders in this builder with the ones * from a previous {@link DataLoaderRegistry} * * @param otherRegistry the previous {@link DataLoaderRegistry} - * * @return this builder for a fluent pattern */ public Builder registerAll(DataLoaderRegistry otherRegistry) { dataLoaders.putAll(otherRegistry.getDataLoadersMap()); + if (otherRegistry instanceof ScheduledDataLoaderRegistry) { + ScheduledDataLoaderRegistry other = (ScheduledDataLoaderRegistry) otherRegistry; + dataLoaderPredicates.putAll(other.dataLoaderPredicates); + } + return this; + } + + /** + * This sets a default predicate on the {@link DataLoaderRegistry} that will control + * whether all {@link DataLoader}s in the {@link DataLoaderRegistry }should be dispatched. + * + * @param dispatchPredicate the predicate + * @return this builder for a fluent pattern + */ + public Builder dispatchPredicate(DispatchPredicate dispatchPredicate) { + this.dispatchPredicate = dispatchPredicate; + return this; + } + + /** + * This sets ticker mode on the registry. When ticker mode is true the registry will + * continuously reschedule the data loaders for possible dispatching after the first call + * to dispatchAll. + * + * @param tickerMode true or false + * @return this builder for a fluent pattern + */ + public Builder tickerMode(boolean tickerMode) { + this.tickerMode = tickerMode; + return this; + } + + public Builder instrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = instrumentation; return this; } @@ -180,6 +368,10 @@ public Builder registerAll(DataLoaderRegistry otherRegistry) { * @return the newly built {@link ScheduledDataLoaderRegistry} */ public ScheduledDataLoaderRegistry build() { + if (scheduledExecutorService == null) { + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + defaultExecutorUsed = true; + } return new ScheduledDataLoaderRegistry(this); } } diff --git a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java new file mode 100644 index 0000000..e7e95d9 --- /dev/null +++ b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java @@ -0,0 +1,95 @@ +package org.dataloader.scheduler; + +import org.dataloader.BatchLoader; +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.MappedBatchLoader; +import org.dataloader.MappedBatchPublisher; +import org.dataloader.BatchPublisher; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionStage; + +/** + * By default, when {@link DataLoader#dispatch()} is called, the {@link BatchLoader} / {@link MappedBatchLoader} function will be invoked + * immediately. However, you can provide your own {@link BatchLoaderScheduler} that allows this call to be done some time into + * the future. You will be passed a callback ({@link ScheduledBatchLoaderCall} / {@link ScheduledMappedBatchLoaderCall} and you are expected + * to eventually call this callback method to make the batch loading happen. + *

+ * Note: Because there is a {@link DataLoaderOptions#maxBatchSize()} it is possible for this scheduling to happen N times for a given {@link DataLoader#dispatch()} + * call. The total set of keys will be sliced into batches themselves and then the {@link BatchLoaderScheduler} will be called for + * each batch of keys. Do not assume that a single call to {@link DataLoader#dispatch()} results in a single call to {@link BatchLoaderScheduler}. + */ +public interface BatchLoaderScheduler { + + + /** + * This represents a callback that will invoke a {@link BatchLoader} function under the covers + * + * @param the value type + */ + interface ScheduledBatchLoaderCall { + CompletionStage> invoke(); + } + + /** + * This represents a callback that will invoke a {@link MappedBatchLoader} function under the covers + * + * @param the key type + * @param the value type + */ + interface ScheduledMappedBatchLoaderCall { + CompletionStage> invoke(); + } + + /** + * This represents a callback that will invoke a {@link BatchPublisher} or {@link MappedBatchPublisher} function under the covers + */ + interface ScheduledBatchPublisherCall { + void invoke(); + } + + /** + * This is called to schedule a {@link BatchLoader} call. + * + * @param scheduledCall the callback that needs to be invoked to allow the {@link BatchLoader} to proceed. + * @param keys this is the list of keys that will be passed to the {@link BatchLoader}. + * This is provided only for informative reasons, and you can't change the keys that are used + * @param environment this is the {@link BatchLoaderEnvironment} in place, + * which can be null if it's a simple {@link BatchLoader} call + * @param the key type + * @param the value type + * + * @return a promise to the values that come from the {@link BatchLoader} + */ + CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); + + /** + * This is called to schedule a {@link MappedBatchLoader} call. + * + * @param scheduledCall the callback that needs to be invoked to allow the {@link MappedBatchLoader} to proceed. + * @param keys this is the list of keys that will be passed to the {@link MappedBatchLoader}. + * This is provided only for informative reasons and, you can't change the keys that are used + * @param environment this is the {@link BatchLoaderEnvironment} in place, + * which can be null if it's a simple {@link MappedBatchLoader} call + * @param the key type + * @param the value type + * + * @return a promise to the values that come from the {@link BatchLoader} + */ + CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); + + /** + * This is called to schedule a {@link BatchPublisher} call. + * + * @param scheduledCall the callback that needs to be invoked to allow the {@link BatchPublisher} to proceed. + * @param keys this is the list of keys that will be passed to the {@link BatchPublisher}. + * This is provided only for informative reasons and, you can't change the keys that are used + * @param environment this is the {@link BatchLoaderEnvironment} in place, + * which can be null if it's a simple {@link BatchPublisher} call + * @param the key type + */ + void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java index f964b29..36ed69a 100644 --- a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java @@ -26,67 +26,67 @@ public DelegatingStatisticsCollector(StatisticsCollector delegateCollector) { } @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { delegateCollector.incrementLoadCount(context); - return collector.incrementLoadCount(context); + collector.incrementLoadCount(context); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { delegateCollector.incrementLoadErrorCount(context); - return collector.incrementLoadErrorCount(context); + collector.incrementLoadErrorCount(context); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { delegateCollector.incrementBatchLoadCountBy(delta, context); - return collector.incrementBatchLoadCountBy(delta, context); + collector.incrementBatchLoadCountBy(delta, context); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { delegateCollector.incrementBatchLoadExceptionCount(context); - return collector.incrementBatchLoadExceptionCount(context); + collector.incrementBatchLoadExceptionCount(context); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { delegateCollector.incrementCacheHitCount(context); - return collector.incrementCacheHitCount(context); + collector.incrementCacheHitCount(context); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } /** - * @return the statistics of the this collector (and not its delegate) + * @return the statistics of the collector (and not its delegate) */ @Override public Statistics getStatistics() { diff --git a/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java b/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java index e7267b3..6397d8f 100644 --- a/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java @@ -14,58 +14,54 @@ public class NoOpStatisticsCollector implements StatisticsCollector { private static final Statistics ZERO_STATS = new Statistics(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return 0; + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return 0; + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - return 0; + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return 0; + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return 0; + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } @Override diff --git a/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java b/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java index 22b3662..2c2898a 100644 --- a/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java @@ -6,7 +6,7 @@ import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; /** * This simple collector uses {@link java.util.concurrent.atomic.AtomicLong}s to collect @@ -15,72 +15,73 @@ * @see org.dataloader.stats.StatisticsCollector */ public class SimpleStatisticsCollector implements StatisticsCollector { - private final AtomicLong loadCount = new AtomicLong(); - private final AtomicLong batchInvokeCount = new AtomicLong(); - private final AtomicLong batchLoadCount = new AtomicLong(); - private final AtomicLong cacheHitCount = new AtomicLong(); - private final AtomicLong batchLoadExceptionCount = new AtomicLong(); - private final AtomicLong loadErrorCount = new AtomicLong(); + + private final LongAdder loadCount = new LongAdder(); + private final LongAdder batchInvokeCount = new LongAdder(); + private final LongAdder batchLoadCount = new LongAdder(); + private final LongAdder cacheHitCount = new LongAdder(); + private final LongAdder batchLoadExceptionCount = new LongAdder(); + private final LongAdder loadErrorCount = new LongAdder(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return loadCount.incrementAndGet(); + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { + loadCount.increment(); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return loadErrorCount.incrementAndGet(); + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + loadErrorCount.increment(); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - batchInvokeCount.incrementAndGet(); - return batchLoadCount.addAndGet(delta); + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + batchInvokeCount.increment(); + batchLoadCount.add(delta); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return batchLoadExceptionCount.incrementAndGet(); + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + batchLoadExceptionCount.increment(); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return cacheHitCount.incrementAndGet(); + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + cacheHitCount.increment(); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } @Override public Statistics getStatistics() { - return new Statistics(loadCount.get(), loadErrorCount.get(), batchInvokeCount.get(), batchLoadCount.get(), batchLoadExceptionCount.get(), cacheHitCount.get()); + return new Statistics(loadCount.sum(), loadErrorCount.sum(), batchInvokeCount.sum(), batchLoadCount.sum(), batchLoadExceptionCount.sum(), cacheHitCount.sum()); } @Override diff --git a/src/main/java/org/dataloader/stats/Statistics.java b/src/main/java/org/dataloader/stats/Statistics.java index 4bc9c69..f5b5e74 100644 --- a/src/main/java/org/dataloader/stats/Statistics.java +++ b/src/main/java/org/dataloader/stats/Statistics.java @@ -54,7 +54,7 @@ public long getLoadCount() { } /** - * @return the number of times the {@link org.dataloader.DataLoader} batch loader function return an specific object that was in error + * @return the number of times the {@link org.dataloader.DataLoader} batch loader function return a specific object that was in error */ public long getLoadErrorCount() { return loadErrorCount; diff --git a/src/main/java/org/dataloader/stats/StatisticsCollector.java b/src/main/java/org/dataloader/stats/StatisticsCollector.java index b32d17e..7b14eab 100644 --- a/src/main/java/org/dataloader/stats/StatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/StatisticsCollector.java @@ -18,21 +18,18 @@ public interface StatisticsCollector { * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return incrementLoadCount(); + default void incrementLoadCount(IncrementLoadCountStatisticsContext context) { + incrementLoadCount(); } /** * Called to increment the number of loads * * @deprecated use {@link #incrementLoadCount(IncrementLoadCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementLoadCount(); + void incrementLoadCount(); /** * Called to increment the number of loads that resulted in an object deemed in error @@ -40,20 +37,18 @@ default long incrementLoadCount(IncrementLoadCountStatisticsContext conte * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation * - * @return the current value after increment */ - default long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return incrementLoadErrorCount(); + default void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + incrementLoadErrorCount(); } /** * Called to increment the number of loads that resulted in an object deemed in error * * @deprecated use {@link #incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementLoadErrorCount(); + void incrementLoadErrorCount(); /** * Called to increment the number of batch loads @@ -61,11 +56,9 @@ default long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContex * @param the class of the key in the data loader * @param delta how much to add to the count * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - return incrementBatchLoadCountBy(delta); + default void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + incrementBatchLoadCountBy(delta); } /** @@ -74,55 +67,48 @@ default long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountBy * @param delta how much to add to the count * * @deprecated use {@link #incrementBatchLoadCountBy(long, IncrementBatchLoadCountByStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementBatchLoadCountBy(long delta); + void incrementBatchLoadCountBy(long delta); /** * Called to increment the number of batch loads exceptions * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return incrementBatchLoadExceptionCount(); + default void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + incrementBatchLoadExceptionCount(); } /** * Called to increment the number of batch loads exceptions * * @deprecated use {@link #incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementBatchLoadExceptionCount(); + void incrementBatchLoadExceptionCount(); /** * Called to increment the number of cache hits * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return incrementCacheHitCount(); + default void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + incrementCacheHitCount(); } /** * Called to increment the number of cache hits * * @deprecated use {@link #incrementCacheHitCount(IncrementCacheHitCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementCacheHitCount(); + void incrementCacheHitCount(); /** - * @return the statistics that have been gathered up to this point in time + * @return the statistics that have been gathered to this point in time */ Statistics getStatistics(); } diff --git a/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java b/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java index d091c5a..cab6d0d 100644 --- a/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java @@ -35,63 +35,63 @@ public ThreadLocalStatisticsCollector resetThread() { } @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { overallCollector.incrementLoadCount(context); - return collector.get().incrementLoadCount(context); + collector.get().incrementLoadCount(context); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { overallCollector.incrementLoadErrorCount(context); - return collector.get().incrementLoadErrorCount(context); + collector.get().incrementLoadErrorCount(context); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { overallCollector.incrementBatchLoadCountBy(delta, context); - return collector.get().incrementBatchLoadCountBy(delta, context); + collector.get().incrementBatchLoadCountBy(delta, context); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { overallCollector.incrementBatchLoadExceptionCount(context); - return collector.get().incrementBatchLoadExceptionCount(context); + collector.get().incrementBatchLoadExceptionCount(context); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { overallCollector.incrementCacheHitCount(context); - return collector.get().incrementCacheHitCount(context); + collector.get().incrementCacheHitCount(context); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } /** diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index e37550e..f391b80 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -1,28 +1,42 @@ import org.dataloader.BatchLoader; import org.dataloader.BatchLoaderEnvironment; import org.dataloader.BatchLoaderWithContext; +import org.dataloader.BatchPublisher; import org.dataloader.CacheMap; import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; +import org.dataloader.DataLoaderRegistry; +import org.dataloader.DispatchResult; import org.dataloader.MappedBatchLoaderWithContext; +import org.dataloader.MappedBatchPublisher; import org.dataloader.Try; import org.dataloader.fixtures.SecurityCtx; import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationContext; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.registries.DispatchPredicate; import org.dataloader.registries.ScheduledDataLoaderRegistry; +import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.Statistics; import org.dataloader.stats.ThreadLocalStatisticsCollector; +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Function; import java.util.stream.Collectors; import static java.lang.String.format; @@ -91,7 +105,7 @@ public CompletionStage> load(List userIds) { private void callContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -106,7 +120,7 @@ public CompletionStage> load(List keys, BatchLoaderEnvironm private void keyContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -166,7 +180,7 @@ private void tryExample() { } } - private void tryBatcLoader() { + private void tryBatchLoader() { DataLoader dataLoader = DataLoaderFactory.newDataLoaderWithTry(new BatchLoader>() { @Override public CompletionStage>> load(List keys) { @@ -182,6 +196,28 @@ public CompletionStage>> load(List keys) { }); } + private void batchPublisher() { + BatchPublisher batchPublisher = new BatchPublisher() { + @Override + public void load(List userIds, Subscriber userSubscriber) { + Publisher userResults = userManager.streamUsersById(userIds); + userResults.subscribe(userSubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); + } + + private void mappedBatchPublisher() { + MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { + @Override + public void load(Set userIds, Subscriber> userEntrySubscriber) { + Publisher> userEntries = userManager.streamUsersById(userIds); + userEntries.subscribe(userEntrySubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); + } + DataLoader userDataLoader; private void clearCacheOnError() { @@ -197,9 +233,10 @@ private void clearCacheOnError() { } BatchLoader userBatchLoader; + BatchLoader teamsBatchLoader; private void disableCache() { - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false).build()); userDataLoader.load("A"); @@ -246,7 +283,7 @@ public CacheMap clear() { private void customCache() { MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache).build(); DataLoaderFactory.newDataLoader(userBatchLoader, options); } @@ -274,11 +311,41 @@ private void statsExample() { private void statsConfigExample() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()).build(); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); } - private void ScheduledDispatche() { + private void snooze(int i) { + } + + private void BatchLoaderSchedulerExample() { + new BatchLoaderScheduler() { + + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(10); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(10); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(10); + scheduledCall.invoke(); + } + }; + } + + private void ScheduledDispatcher() { DispatchPredicate depthOrTimePredicate = DispatchPredicate.dispatchIfDepthGreaterThan(10) .or(DispatchPredicate.dispatchIfLongerThan(Duration.ofMillis(200))); @@ -288,4 +355,94 @@ private void ScheduledDispatche() { .register("users", userDataLoader) .build(); } + + + DataLoader dataLoaderA = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader dataLoaderB = DataLoaderFactory.newDataLoader(keys -> { + return CompletableFuture.completedFuture(Collections.singletonList(1L)); + }); + + private void ScheduledDispatcherChained() { + CompletableFuture chainedCalls = dataLoaderA.load("user1") + .thenCompose(userAsKey -> dataLoaderB.load(userAsKey)); + + + CompletableFuture chainedWithImmediateDispatch = dataLoaderA.load("user1") + .thenCompose(userAsKey -> { + CompletableFuture loadB = dataLoaderB.load(userAsKey); + dataLoaderB.dispatch(); + return loadB; + }); + + + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dataLoaderA) + .register("b", dataLoaderB) + .scheduledExecutorService(executorService) + .schedule(Duration.ofMillis(10)) + .tickerMode(true) // ticker mode is on + .build(); + + } + + private DataLoaderInstrumentation timingInstrumentation = DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION; + + private void instrumentationExample() { + + DataLoaderInstrumentation timingInstrumentation = new DataLoaderInstrumentation() { + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("dispatch time: %d ms", ms)); + }); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("batch loader time: %d ms", ms)); + }); + } + }; + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation).build(); + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); + } + + private void registryExample() { + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoader changedUsersDataLoader = registry.getDataLoader("users"); + + } + + private void combiningRegistryExample() { + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoaderRegistry registryCombined = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .registerAll(registry) + .build(); + + DataLoader changedUsersDataLoader = registryCombined.getDataLoader("users"); + + } } diff --git a/src/test/java/org/dataloader/ClockDataLoader.java b/src/test/java/org/dataloader/ClockDataLoader.java index 21faeea..0c83316 100644 --- a/src/test/java/org/dataloader/ClockDataLoader.java +++ b/src/test/java/org/dataloader/ClockDataLoader.java @@ -9,7 +9,7 @@ public ClockDataLoader(Object batchLoadFunction, Clock clock) { } public ClockDataLoader(Object batchLoadFunction, DataLoaderOptions options, Clock clock) { - super(batchLoadFunction, options, clock); + super(null, batchLoadFunction, options, clock); } } diff --git a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java index 36e0ed4..274820d 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java @@ -1,11 +1,8 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -14,8 +11,8 @@ import static java.util.Collections.singletonList; import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; /** * Tests related to context. DataLoaderTest is getting to big and needs refactoring @@ -44,32 +41,40 @@ public void context_is_passed_to_batch_loader_function() { return CompletableFuture.completedFuture(list); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx").build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A"); loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx", "B-ctx", "C-ctx", "D-ctx"))); + assertThat(results, equalTo(asList("A-ctx", "B-ctx", "C-ctx", "D-ctx", "E-ctx", "F-ctx"))); } @Test public void key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx").build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); loader.load("B", "bCtx"); loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", "fCtx"); + loader.loadMany(keysAndContexts); List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:fCtx-l:fCtx"))); } @Test @@ -77,33 +82,45 @@ public void key_contexts_are_passed_to_batch_loader_function_when_batching_disab BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() .setBatchingEnabled(false) - .setBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newDataLoader(batchLoader, options); CompletableFuture aLoad = loader.load("A", "aCtx"); CompletableFuture bLoad = loader.load("B", "bCtx"); - CompletableFuture> canDLoad = loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + CompletableFuture> cAndDLoad = loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", "fCtx"); + CompletableFuture> eAndFLoad = loader.loadMany(keysAndContexts); List results = new ArrayList<>(asList(aLoad.join(), bLoad.join())); - results.addAll(canDLoad.join()); + results.addAll(cAndDLoad.join()); + results.addAll(eAndFLoad.join().values()); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:fCtx-l:fCtx"))); } @Test public void missing_key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); loader.load("B"); loader.loadMany(asList("C", "D"), singletonList("cCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:null-l:null", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:null-l:null"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:null-l:null", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:null-l:null", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:null-l:null"))); } @Test @@ -118,16 +135,22 @@ public void context_is_passed_to_map_batch_loader_function() { return CompletableFuture.completedFuture(map); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newMappedDataLoader(mapBatchLoader, options); loader.load("A", "aCtx"); loader.load("B"); loader.loadMany(asList("C", "D"), singletonList("cCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-aCtx", "B-ctx-null", "C-ctx-cCtx", "D-ctx-null"))); + assertThat(results, equalTo(asList("A-ctx-aCtx", "B-ctx-null", "C-ctx-cCtx", "D-ctx-null", "E-ctx-eCtx", "F-ctx-null"))); } @Test @@ -142,9 +165,14 @@ public void null_is_passed_as_context_if_you_do_nothing() { loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null"))); + assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null", "E-null", "F-null"))); } @Test @@ -160,9 +188,14 @@ public void null_is_passed_as_context_to_map_loader_if_you_do_nothing() { loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null"))); + assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null", "E-null", "F-null"))); } @Test @@ -170,7 +203,8 @@ public void mmap_semantics_apply_to_batch_loader_context() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() .setBatchLoaderContextProvider(() -> "ctx") - .setCachingEnabled(false); + .setCachingEnabled(false) + .build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); diff --git a/src/test/java/org/dataloader/DataLoaderBuilderTest.java b/src/test/java/org/dataloader/DataLoaderBuilderTest.java new file mode 100644 index 0000000..bf8b762 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderBuilderTest.java @@ -0,0 +1,76 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class DataLoaderBuilderTest { + + BatchLoader batchLoader1 = keys -> null; + + BatchLoader batchLoader2 = keys -> null; + + DataLoaderOptions defaultOptions = DataLoaderOptions.newOptions().build(); + DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().setCachingEnabled(false).build(); + + @Test + void canBuildNewDataLoaders() { + DataLoaderFactory.Builder builder = DataLoaderFactory.builder(); + builder.options(differentOptions); + builder.batchLoadFunction(batchLoader1); + DataLoader dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(differentOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // and we can copy ok + // + builder = DataLoaderFactory.builder(dataLoader); + dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(differentOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // and we can copy and transform ok + // + builder = DataLoaderFactory.builder(dataLoader); + builder.options(defaultOptions); + builder.batchLoadFunction(batchLoader2); + dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader2)); + } + + @Test + void theDataLoaderCanTransform() { + DataLoader dataLoaderOrig = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); + assertThat(dataLoaderOrig.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoaderOrig.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // we can transform the data loader + // + DataLoader dataLoaderTransformed = dataLoaderOrig.transform(it -> { + it.options(differentOptions); + it.batchLoadFunction(batchLoader2); + }); + + assertThat(dataLoaderTransformed, not(equalTo(dataLoaderOrig))); + assertThat(dataLoaderTransformed.getOptions(), equalTo(differentOptions)); + assertThat(dataLoaderTransformed.getBatchLoadFunction(), equalTo(batchLoader2)); + + // can copy values + dataLoaderOrig = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); + + dataLoaderTransformed = dataLoaderOrig.transform(it -> { + it.batchLoadFunction(batchLoader2); + }); + + assertThat(dataLoaderTransformed, not(equalTo(dataLoaderOrig))); + assertThat(dataLoaderTransformed.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoaderTransformed.getBatchLoadFunction(), equalTo(batchLoader2)); + + } +} diff --git a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java index abfc8d3..df364a2 100644 --- a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java +++ b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java @@ -1,6 +1,6 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; @@ -8,8 +8,8 @@ import java.util.concurrent.CompletableFuture; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; /** * Tests for cacheMap functionality.. @@ -43,7 +43,7 @@ public void should_access_to_future_dependants() { Collection> futures = dataLoader.getCacheMap().getAll(); List> futuresList = new ArrayList<>(futures); - assertThat(futuresList.get(0).getNumberOfDependents(), equalTo(2)); - assertThat(futuresList.get(1).getNumberOfDependents(), equalTo(1)); + assertThat(futuresList.get(0).getNumberOfDependents(), equalTo(4)); // instrumentation is depending on the CF completing + assertThat(futuresList.get(1).getNumberOfDependents(), equalTo(2)); } } diff --git a/src/test/java/org/dataloader/DataLoaderFactoryTest.java b/src/test/java/org/dataloader/DataLoaderFactoryTest.java new file mode 100644 index 0000000..3b3f368 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderFactoryTest.java @@ -0,0 +1,51 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class DataLoaderFactoryTest { + + @Test + void can_create_via_builder() { + BatchLoaderWithContext loader = (keys, environment) -> CompletableFuture.completedFuture(keys); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchingEnabled(true).build(); + + DataLoader dl = DataLoaderFactory.builder() + .name("x").batchLoader(loader).options(options).build(); + + assertNotNull(dl.getName()); + assertThat(dl.getName(), equalTo("x")); + assertThat(dl.getBatchLoadFunction(), equalTo(loader)); + assertThat(dl.getOptions(), equalTo(options)); + + BatchLoaderWithContext> loaderTry = (keys, environment) + -> CompletableFuture.completedFuture(keys.stream().map(Try::succeeded).collect(Collectors.toList())); + + DataLoader> dlTry = DataLoaderFactory.>builder() + .name("try").batchLoader(loaderTry).options(options).build(); + + assertNotNull(dlTry.getName()); + assertThat(dlTry.getName(), equalTo("try")); + assertThat(dlTry.getBatchLoadFunction(), equalTo(loaderTry)); + assertThat(dlTry.getOptions(), equalTo(options)); + + MappedBatchLoader> mappedLoaderTry = (keys) + -> CompletableFuture.completedFuture( + keys.stream().collect(Collectors.toMap(k -> k, Try::succeeded)) + ); + + DataLoader> dlTry2 = DataLoaderFactory.>builder() + .name("try2").mappedBatchLoader(mappedLoaderTry).options(options).build(); + + assertNotNull(dlTry2.getName()); + assertThat(dlTry2.getName(), equalTo("try2")); + assertThat(dlTry2.getBatchLoadFunction(), equalTo(mappedLoaderTry)); + assertThat(dlTry2.getOptions(), equalTo(options)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java index 916fdef..f0a50d6 100644 --- a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java +++ b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java @@ -1,15 +1,15 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; /** * Tests for IfPresent and IfCompleted functionality. @@ -34,10 +34,10 @@ public void should_detect_if_present_cf() { assertThat(cachedPromise.get(), sameInstance(future1)); - // but its not done! + // but it's not done! assertThat(cachedPromise.get().isDone(), equalTo(false)); // - // and hence it cant be loaded as complete + // and hence it can't be loaded as complete cachedPromise = dataLoader.getIfCompleted(1); assertThat(cachedPromise.isPresent(), equalTo(false)); } diff --git a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java deleted file mode 100644 index 0fced79..0000000 --- a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java +++ /dev/null @@ -1,184 +0,0 @@ -package org.dataloader; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.newDataLoader; -import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.futureError; -import static org.dataloader.fixtures.TestKit.listFrom; -import static org.dataloader.impl.CompletableFutureKit.cause; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -/** - * Much of the tests that related to {@link MappedBatchLoader} also related to - * {@link org.dataloader.BatchLoader}. This is white box testing somewhat because we could have repeated - * ALL the tests in {@link org.dataloader.DataLoaderTest} here as well but chose not to because we KNOW that - * DataLoader differs only a little in how it handles the 2 types of loader functions. We choose to grab some - * common functionality for repeat testing and otherwise rely on the very complete other tests. - */ -public class DataLoaderMapBatchLoaderTest { - - MappedBatchLoader evensOnlyMappedBatchLoader = (keys) -> { - Map mapOfResults = new HashMap<>(); - - AtomicInteger index = new AtomicInteger(); - keys.forEach(k -> { - int i = index.getAndIncrement(); - if (i % 2 == 0) { - mapOfResults.put(k, k); - } - }); - return CompletableFuture.completedFuture(mapOfResults); - }; - - private static DataLoader idMapLoader(DataLoaderOptions options, List> loadCalls) { - MappedBatchLoader kvBatchLoader = (keys) -> { - loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - //noinspection unchecked - keys.forEach(k -> map.put(k, (V) k)); - return CompletableFuture.completedFuture(map); - }; - return DataLoaderFactory.newMappedDataLoader(kvBatchLoader, options); - } - - private static DataLoader idMapLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader((keys) -> { - loadCalls.add(new ArrayList<>(keys)); - return futureError(); - }, options); - } - - - @Test - public void basic_map_batch_loading() { - DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); - - loader.load("A"); - loader.load("B"); - loader.loadMany(asList("C", "D")); - - List results = loader.dispatchAndJoin(); - - assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", null, "C", null))); - } - - - @Test - public void should_map_Batch_multiple_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load(1); - CompletableFuture future2 = identityLoader.load(2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo(1)); - assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void can_split_max_batch_sizes_correctly() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(newOptions().setMaxBatchSize(5), loadCalls); - - for (int i = 0; i < 21; i++) { - identityLoader.load(i); - } - List> expectedCalls = new ArrayList<>(); - expectedCalls.add(listFrom(0, 5)); - expectedCalls.add(listFrom(5, 10)); - expectedCalls.add(listFrom(10, 15)); - expectedCalls.add(listFrom(15, 20)); - expectedCalls.add(listFrom(20, 21)); - - List result = identityLoader.dispatch().join(); - - assertThat(result, equalTo(listFrom(0, 21))); - assertThat(loadCalls, equalTo(expectedCalls)); - } - - @Test - public void should_Propagate_error_to_all_loads() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idMapLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - CompletableFuture future2 = errorLoader.load(2); - errorLoader.dispatch(); - - await().until(future1::isDone); - - assertThat(future1.isCompletedExceptionally(), is(true)); - Throwable cause = cause(future1); - assert cause != null; - assertThat(cause, instanceOf(IllegalStateException.class)); - assertThat(cause.getMessage(), equalTo("Error")); - - await().until(future2::isDone); - cause = cause(future2); - assert cause != null; - assertThat(cause.getMessage(), equalTo(cause.getMessage())); - - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(false), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - - // the map batch functions use a set of keys as input and hence remove duplicates unlike list variant - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(true), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - -} diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java new file mode 100644 index 0000000..81a7126 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -0,0 +1,226 @@ +package org.dataloader; + +import org.dataloader.impl.DefaultCacheMap; +import org.dataloader.impl.NoOpValueCache; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.scheduler.BatchLoaderScheduler; +import org.dataloader.stats.NoOpStatisticsCollector; +import org.dataloader.stats.StatisticsCollector; +import org.hamcrest.CoreMatchers; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +class DataLoaderOptionsTest { + + DataLoaderOptions optionsDefault = new DataLoaderOptions(); + + @Test + void canCreateDefaultOptions() { + + assertThat(optionsDefault.batchingEnabled(), equalTo(true)); + assertThat(optionsDefault.cachingEnabled(), equalTo(true)); + assertThat(optionsDefault.cachingExceptionsEnabled(), equalTo(true)); + assertThat(optionsDefault.maxBatchSize(), equalTo(-1)); + assertThat(optionsDefault.getBatchLoaderScheduler(), equalTo(null)); + + DataLoaderOptions builtOptions = DataLoaderOptions.newDefaultOptions(); + assertThat(builtOptions, equalTo(optionsDefault)); + assertThat(builtOptions == optionsDefault, equalTo(false)); + + DataLoaderOptions transformedOptions = optionsDefault.transform(builder -> { + }); + assertThat(transformedOptions, equalTo(optionsDefault)); + assertThat(transformedOptions == optionsDefault, equalTo(false)); + } + + @Test + void canCopyOk() { + DataLoaderOptions optionsNext = DataLoaderOptions.newOptions(optionsDefault).build(); + assertThat(optionsNext, equalTo(optionsDefault)); + assertThat(optionsNext == optionsDefault, equalTo(false)); + } + + BatchLoaderScheduler testBatchLoaderScheduler = new BatchLoaderScheduler() { + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return null; + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return null; + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + + } + }; + + BatchLoaderContextProvider testBatchLoaderContextProvider = () -> null; + + CacheMap testCacheMap = new DefaultCacheMap<>(); + + ValueCache testValueCache = new NoOpValueCache<>(); + + CacheKey testCacheKey = new CacheKey() { + @Override + public Object getKey(Object input) { + return null; + } + }; + + ValueCacheOptions testValueCacheOptions = ValueCacheOptions.newOptions(); + + NoOpStatisticsCollector noOpStatisticsCollector = new NoOpStatisticsCollector(); + Supplier testStatisticsCollectorSupplier = () -> noOpStatisticsCollector; + + @Test + void canBuildOk() { + assertThat(optionsDefault.transform(b -> b.setBatchingEnabled(false)).batchingEnabled(), + equalTo(false)); + assertThat(optionsDefault.transform(b -> b.setBatchLoaderScheduler(testBatchLoaderScheduler)).getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(optionsDefault.transform(b -> b.setBatchLoaderContextProvider(testBatchLoaderContextProvider)).getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(optionsDefault.transform(b -> b.setCacheMap(testCacheMap)).cacheMap().get(), + equalTo(testCacheMap)); + assertThat(optionsDefault.transform(b -> b.setCachingEnabled(false)).cachingEnabled(), + equalTo(false)); + assertThat(optionsDefault.transform(b -> b.setValueCacheOptions(testValueCacheOptions)).getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(optionsDefault.transform(b -> b.setCacheKeyFunction(testCacheKey)).cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(optionsDefault.transform(b -> b.setValueCache(testValueCache)).valueCache().get(), + equalTo(testValueCache)); + assertThat(optionsDefault.transform(b -> b.setMaxBatchSize(10)).maxBatchSize(), + equalTo(10)); + assertThat(optionsDefault.transform(b -> b.setStatisticsCollector(testStatisticsCollectorSupplier)).getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + + DataLoaderOptions builtOptions = optionsDefault.transform(builder -> { + builder.setBatchingEnabled(false); + builder.setCachingExceptionsEnabled(false); + builder.setCachingEnabled(false); + builder.setBatchLoaderScheduler(testBatchLoaderScheduler); + builder.setBatchLoaderContextProvider(testBatchLoaderContextProvider); + builder.setCacheMap(testCacheMap); + builder.setValueCache(testValueCache); + builder.setCacheKeyFunction(testCacheKey); + builder.setValueCacheOptions(testValueCacheOptions); + builder.setMaxBatchSize(10); + builder.setStatisticsCollector(testStatisticsCollectorSupplier); + }); + + assertThat(builtOptions.batchingEnabled(), + equalTo(false)); + assertThat(builtOptions.getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(builtOptions.getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(builtOptions.cacheMap().get(), + equalTo(testCacheMap)); + assertThat(builtOptions.cachingEnabled(), + equalTo(false)); + assertThat(builtOptions.getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(builtOptions.cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(builtOptions.valueCache().get(), + equalTo(testValueCache)); + assertThat(builtOptions.maxBatchSize(), + equalTo(10)); + assertThat(builtOptions.getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + + } + + @Test + void canBuildViaBuilderOk() { + + DataLoaderOptions.Builder builder = DataLoaderOptions.newOptions(); + builder.setBatchingEnabled(false); + builder.setCachingExceptionsEnabled(false); + builder.setCachingEnabled(false); + builder.setBatchLoaderScheduler(testBatchLoaderScheduler); + builder.setBatchLoaderContextProvider(testBatchLoaderContextProvider); + builder.setCacheMap(testCacheMap); + builder.setValueCache(testValueCache); + builder.setCacheKeyFunction(testCacheKey); + builder.setValueCacheOptions(testValueCacheOptions); + builder.setMaxBatchSize(10); + builder.setStatisticsCollector(testStatisticsCollectorSupplier); + + DataLoaderOptions builtOptions = builder.build(); + + assertThat(builtOptions.batchingEnabled(), + equalTo(false)); + assertThat(builtOptions.getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(builtOptions.getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(builtOptions.cacheMap().get(), + equalTo(testCacheMap)); + assertThat(builtOptions.cachingEnabled(), + equalTo(false)); + assertThat(builtOptions.getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(builtOptions.cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(builtOptions.valueCache().get(), + equalTo(testValueCache)); + assertThat(builtOptions.maxBatchSize(), + equalTo(10)); + assertThat(builtOptions.getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + } + + @Test + void canCopyExistingOptionValuesOnTransform() { + + DataLoaderInstrumentation instrumentation1 = new DataLoaderInstrumentation() { + }; + DataLoaderInstrumentation instrumentation2 = new DataLoaderInstrumentation() { + }; + BatchLoaderContextProvider contextProvider1 = () -> null; + + DataLoaderOptions startingOptions = DataLoaderOptions.newOptions().setBatchingEnabled(false) + .setCachingEnabled(false) + .setInstrumentation(instrumentation1) + .setBatchLoaderContextProvider(contextProvider1) + .build(); + + assertThat(startingOptions.batchingEnabled(), equalTo(false)); + assertThat(startingOptions.cachingEnabled(), equalTo(false)); + assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); + assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + DataLoaderOptions newOptions = startingOptions.transform(builder -> + builder.setBatchingEnabled(true).setInstrumentation(instrumentation2)); + + + // immutable + assertThat(newOptions, CoreMatchers.not(startingOptions)); + assertThat(startingOptions.batchingEnabled(), equalTo(false)); + assertThat(startingOptions.cachingEnabled(), equalTo(false)); + assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); + assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + // stayed the same + assertThat(newOptions.cachingEnabled(), equalTo(false)); + assertThat(newOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + // was changed + assertThat(newOptions.batchingEnabled(), equalTo(true)); + assertThat(newOptions.getInstrumentation(), equalTo(instrumentation2)); + + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index aeaf668..270bd50 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -1,26 +1,30 @@ package org.dataloader; +import org.dataloader.impl.DataLoaderAssertionException; import org.dataloader.stats.SimpleStatisticsCollector; import org.dataloader.stats.Statistics; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import java.util.List; import java.util.concurrent.CompletableFuture; import static java.util.Arrays.asList; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; public class DataLoaderRegistryTest { final BatchLoader identityBatchLoader = CompletableFuture::completedFuture; @Test public void registration_works() { - DataLoader dlA = newDataLoader(identityBatchLoader); - DataLoader dlB = newDataLoader(identityBatchLoader); - DataLoader dlC = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); + DataLoader dlB = newDataLoader("b", identityBatchLoader); + DataLoader dlC = newDataLoader("c", identityBatchLoader); + DataLoader dlUnnamed = newDataLoader(identityBatchLoader); DataLoaderRegistry registry = new DataLoaderRegistry(); @@ -40,7 +44,7 @@ public void registration_works() { // and unregister (fluently) DataLoaderRegistry dlR = registry.unregister("c"); - assertThat(dlR,equalTo(registry)); + assertThat(dlR, equalTo(registry)); assertThat(registry.getDataLoaders(), equalTo(asList(dlA, dlB))); @@ -49,15 +53,27 @@ public void registration_works() { assertThat(readDL, sameInstance(dlA)); assertThat(registry.getKeys(), hasItems("a", "b")); + + + // named registry + registry = new DataLoaderRegistry(); + registry.register(dlA); + assertThat(registry.getDataLoaders(), equalTo(List.of(dlA))); + + try { + registry.register(dlUnnamed); + Assertions.fail("Should have thrown an exception"); + } catch (DataLoaderAssertionException ignored) { + } } @Test public void registries_can_be_combined() { - DataLoader dlA = newDataLoader(identityBatchLoader); - DataLoader dlB = newDataLoader(identityBatchLoader); - DataLoader dlC = newDataLoader(identityBatchLoader); - DataLoader dlD = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); + DataLoader dlB = newDataLoader("b", identityBatchLoader); + DataLoader dlC = newDataLoader("c", identityBatchLoader); + DataLoader dlD = newDataLoader("d", identityBatchLoader); DataLoaderRegistry registry1 = new DataLoaderRegistry(); @@ -79,17 +95,21 @@ public void stats_can_be_collected() { DataLoaderRegistry registry = new DataLoaderRegistry(); DataLoader dlA = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); DataLoader dlB = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); DataLoader dlC = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); registry.register("a", dlA).register("b", dlB).register("c", dlC); + dlA = registry.getDataLoader("a"); + dlB = registry.getDataLoader("b"); + dlC = registry.getDataLoader("b"); + dlA.load("X"); dlB.load("Y"); dlC.load("Z"); @@ -116,7 +136,7 @@ public void computeIfAbsent_creates_a_data_loader_if_there_was_no_value_at_key() DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); DataLoader registered = registry.computeIfAbsent("a", (key) -> dlA); assertThat(registered, equalTo(dlA)); @@ -129,11 +149,11 @@ public void computeIfAbsent_returns_an_existing_data_loader_if_there_was_a_value DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); registry.computeIfAbsent("a", (key) -> dlA); // register again at same key - DataLoader dlA2 = newDataLoader(identityBatchLoader); + DataLoader dlA2 = newDataLoader("a", identityBatchLoader); DataLoader registered = registry.computeIfAbsent("a", (key) -> dlA2); assertThat(registered, equalTo(dlA)); @@ -149,8 +169,8 @@ public void dispatch_counts_are_maintained() { DataLoader dlA = newDataLoader(identityBatchLoader); DataLoader dlB = newDataLoader(identityBatchLoader); - registry.register("a", dlA); - registry.register("b", dlB); + dlA = registry.registerAndGet("a", dlA); + dlB = registry.registerAndGet("b", dlB); dlA.load("av1"); dlA.load("av2"); diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index a76d0f7..04ea2e7 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -9,19 +9,20 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.assertThat; /** * Tests related to stats. DataLoaderTest is getting to big and needs refactoring @@ -32,7 +33,7 @@ public class DataLoaderStatsTest { public void stats_are_collected_by_default() { BatchLoader batchLoader = CompletableFuture::completedFuture; DataLoader loader = newDataLoader(batchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); loader.load("A"); @@ -68,13 +69,13 @@ public void stats_are_collected_by_default() { @Test public void stats_are_collected_with_specified_collector() { - // lets prime it with some numbers so we know its ours + // let's prime it with some numbers, so we know it's ours StatisticsCollector collector = new SimpleStatisticsCollector(); collector.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(1, null)); collector.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(1, null)); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).build(); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); @@ -112,15 +113,16 @@ public void stats_are_collected_with_caching_disabled() { StatisticsCollector collector = new SimpleStatisticsCollector(); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).setCachingEnabled(false); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).setCachingEnabled(false).build(); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); loader.load("B"); loader.loadMany(asList("C", "D")); + loader.loadMany(Map.of("E", "E", "F", "F")); Statistics stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(4L)); + assertThat(stats.getLoadCount(), equalTo(6L)); assertThat(stats.getBatchInvokeCount(), equalTo(0L)); assertThat(stats.getBatchLoadCount(), equalTo(0L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); @@ -128,9 +130,9 @@ public void stats_are_collected_with_caching_disabled() { loader.dispatch(); stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(4L)); + assertThat(stats.getLoadCount(), equalTo(6L)); assertThat(stats.getBatchInvokeCount(), equalTo(1L)); - assertThat(stats.getBatchLoadCount(), equalTo(4L)); + assertThat(stats.getBatchLoadCount(), equalTo(6L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); loader.load("A"); @@ -139,9 +141,9 @@ public void stats_are_collected_with_caching_disabled() { loader.dispatch(); stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(6L)); + assertThat(stats.getLoadCount(), equalTo(8L)); assertThat(stats.getBatchInvokeCount(), equalTo(2L)); - assertThat(stats.getBatchLoadCount(), equalTo(6L)); + assertThat(stats.getBatchLoadCount(), equalTo(8L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); } @@ -164,7 +166,7 @@ public void stats_are_collected_with_caching_disabled() { @Test public void stats_are_collected_on_exceptions() { DataLoader loader = DataLoaderFactory.newDataLoaderWithTry(batchLoaderThatBlows, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); loader.load("A"); @@ -219,63 +221,55 @@ private static class ContextPassingStatisticsCollector implements StatisticsColl public List> incrementCacheHitCountStatisticsContexts = new ArrayList<>(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { incrementLoadCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementLoadCount() { - return 0; + public void incrementLoadCount() { + } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { incrementLoadErrorCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementLoadErrorCount() { - return 0; + public void incrementLoadErrorCount() { + } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { incrementBatchLoadCountByStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return 0; + public void incrementBatchLoadCountBy(long delta) { } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { incrementBatchLoadExceptionCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return 0; + public void incrementBatchLoadExceptionCount() { } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { incrementCacheHitCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementCacheHitCount() { - return 0; + public void incrementCacheHitCount() { } @Override @@ -288,7 +282,7 @@ public Statistics getStatistics() { public void context_is_passed_through_to_collector() { ContextPassingStatisticsCollector statisticsCollector = new ContextPassingStatisticsCollector(); DataLoader> loader = newDataLoader(batchLoaderThatBlows, - DataLoaderOptions.newOptions().setStatisticsCollector(() -> statisticsCollector) + DataLoaderOptions.newOptions().setStatisticsCollector(() -> statisticsCollector).build() ); loader.load("key", "keyContext"); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 63a834d..224b54d 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -16,38 +16,60 @@ package org.dataloader; +import org.awaitility.Duration; import org.dataloader.fixtures.CustomCacheMap; import org.dataloader.fixtures.JsonObject; -import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; +import org.dataloader.fixtures.parameterized.ListDataLoaderFactory; +import org.dataloader.fixtures.parameterized.MappedDataLoaderFactory; +import org.dataloader.fixtures.parameterized.MappedPublisherDataLoaderFactory; +import org.dataloader.fixtures.parameterized.PublisherDataLoaderFactory; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.dataloader.fixtures.parameterized.TestReactiveDataLoaderFactory; import org.dataloader.impl.CompletableFutureKit; -import org.junit.Test; +import org.dataloader.impl.DataLoaderAssertionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static java.util.concurrent.CompletableFuture.supplyAsync; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderOptions.newDefaultOptions; import static org.dataloader.DataLoaderOptions.newOptions; +import static org.dataloader.fixtures.TestKit.areAllDone; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Tests for {@link DataLoader}. @@ -64,7 +86,7 @@ public class DataLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = newDataLoader(CompletableFuture::completedFuture); CompletionStage future1 = identityLoader.load(1); @@ -77,9 +99,56 @@ public void should_Build_a_really_really_simple_data_loader() { } @Test - public void should_Support_loading_multiple_keys_in_one_call() { + public void should_Build_a_named_data_loader() { + BatchLoader loadFunction = CompletableFuture::completedFuture; + DataLoader dl = newDataLoader("name", loadFunction, DataLoaderOptions.newDefaultOptions()); + + assertNotNull(dl.getName()); + assertThat(dl.getName(), equalTo("name")); + + DataLoader dl2 = DataLoaderFactory.builder().name("name2").batchLoader(loadFunction).build(); + + assertNotNull(dl2.getName()); + assertThat(dl2.getName(), equalTo("name2")); + } + + @Test + public void basic_map_batch_loading() { + MappedBatchLoader evensOnlyMappedBatchLoader = (keys) -> { + Map mapOfResults = new HashMap<>(); + + AtomicInteger index = new AtomicInteger(); + keys.forEach(k -> { + int i = index.getAndIncrement(); + if (i % 2 == 0) { + mapOfResults.put(k, k); + } + }); + return completedFuture(mapOfResults); + }; + DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); + + final List keys = asList("C", "D"); + final Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + + loader.load("A"); + loader.load("B"); + loader.loadMany(keys); + loader.loadMany(keysAndContexts); + + List results = loader.dispatchAndJoin(); + + assertThat(results.size(), equalTo(6)); + assertThat(results, equalTo(asList("A", null, "C", null, "E", null))); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Support_loading_multiple_keys_in_one_call_via_list(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); futureAll.thenAccept(promisedValues -> { @@ -91,10 +160,31 @@ public void should_Support_loading_multiple_keys_in_one_call() { assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); } - @Test - public void should_Resolve_to_empty_list_when_no_keys_supplied() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Support_loading_multiple_keys_in_one_call_via_map(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + + final Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put(1, null); + keysAndContexts.put(2, null); + + CompletionStage> futureAll = identityLoader.loadMany(keysAndContexts); + futureAll.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(2)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureAll.toCompletableFuture().join(), equalTo(Map.of(1, 1, 2, 2))); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); futureEmpty.thenAccept(promisedValues -> { assertThat(promisedValues.size(), is(0)); @@ -105,10 +195,26 @@ public void should_Resolve_to_empty_list_when_no_keys_supplied() { assertThat(futureEmpty.join(), empty()); } - @Test - public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Resolve_to_empty_map_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyMap()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureEmpty.join(), anEmptyMap()); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_list(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); futureEmpty.thenAccept(promisedValues -> { assertThat(promisedValues.size(), is(0)); @@ -119,10 +225,26 @@ public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { assertThat(dispatchResult.getKeysCount(), equalTo(0)); } - @Test - public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_map(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyMap()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + await().untilAtomic(success, is(true)); + assertThat(dispatchResult.getKeysCount(), equalTo(0)); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Batch_multiple_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); @@ -134,24 +256,26 @@ public void should_Batch_multiple_requests() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } - @Test - public void should_Return_number_of_batched_entries() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Return_number_of_batched_entries(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); await().until(() -> future1.isDone() && future2.isDone()); - assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because its the number dispatched (by key) not the load calls + assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because it's the number dispatched (by key) not the load calls assertThat(dispatchResult.getPromisedResults().isDone(), equalTo(true)); } - @Test - public void should_Coalesce_identical_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Coalesce_identical_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1a = identityLoader.load(1); CompletableFuture future1b = identityLoader.load(1); @@ -164,10 +288,11 @@ public void should_Coalesce_identical_requests() throws ExecutionException, Inte assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); } - @Test - public void should_Cache_repeated_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Cache_repeated_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -199,10 +324,11 @@ public void should_Cache_repeated_requests() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); } - @Test - public void should_Not_redispatch_previous_load() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Not_redispatch_previous_load(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); identityLoader.dispatch(); @@ -216,10 +342,11 @@ public void should_Not_redispatch_previous_load() throws ExecutionException, Int assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); } - @Test - public void should_Cache_on_redispatch() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Cache_on_redispatch(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); identityLoader.dispatch(); @@ -227,16 +354,24 @@ public void should_Cache_on_redispatch() throws ExecutionException, InterruptedE CompletableFuture> future2 = identityLoader.loadMany(asList("A", "B")); identityLoader.dispatch(); - await().until(() -> future1.isDone() && future2.isDone()); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("A", null); + keysAndContexts.put("C", null); + CompletableFuture> future3 = identityLoader.loadMany(keysAndContexts); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo(asList("A", "B"))); - assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); + assertThat(future3.get(), equalTo(keysAndContexts.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey)))); + assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B"), singletonList("C")))); } - @Test - public void should_Clear_single_value_in_loader() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Clear_single_value_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -261,10 +396,11 @@ public void should_Clear_single_value_in_loader() throws ExecutionException, Int assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("A")))); } - @Test - public void should_Clear_all_values_in_loader() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Clear_all_values_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -288,10 +424,11 @@ public void should_Clear_all_values_in_loader() throws ExecutionException, Inter assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "B")))); } - @Test - public void should_Allow_priming_the_cache() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Allow_priming_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); DataLoader dlFluency = identityLoader.prime("A", "A"); assertThat(dlFluency, equalTo(identityLoader)); @@ -306,10 +443,11 @@ public void should_Allow_priming_the_cache() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Not_prime_keys_that_already_exist() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Not_prime_keys_that_already_exist(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime("A", "X"); @@ -334,10 +472,11 @@ public void should_Not_prime_keys_that_already_exist() throws ExecutionException assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Allow_to_forcefully_prime_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime("A", "X"); @@ -362,12 +501,13 @@ public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionExcepti assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Allow_priming_the_cache_with_a_future() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Allow_priming_the_cache_with_a_future(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); - DataLoader dlFluency = identityLoader.prime("A", CompletableFuture.completedFuture("A")); + DataLoader dlFluency = identityLoader.prime("A", completedFuture("A")); assertThat(dlFluency, equalTo(identityLoader)); CompletableFuture future1 = identityLoader.load("A"); @@ -380,10 +520,11 @@ public void should_Allow_priming_the_cache_with_a_future() throws ExecutionExcep assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_not_Cache_failed_fetches_on_complete_failure() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_not_Cache_failed_fetches_on_complete_failure(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -401,10 +542,11 @@ public void should_not_Cache_failed_fetches_on_complete_failure() { assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); } - @Test - public void should_Resolve_to_error_to_indicate_failure() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Resolve_to_error_to_indicate_failure(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + DataLoader evenLoader = factory.idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = evenLoader.load(1); evenLoader.dispatch(); @@ -423,11 +565,12 @@ public void should_Resolve_to_error_to_indicate_failure() throws ExecutionExcept // Accept any kind of key. - @Test - public void should_Represent_failures_and_successes_simultaneously() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Represent_failures_and_successes_simultaneously(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { AtomicBoolean success = new AtomicBoolean(); List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + DataLoader evenLoader = factory.idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = evenLoader.load(1); CompletableFuture future2 = evenLoader.load(2); @@ -449,10 +592,11 @@ public void should_Represent_failures_and_successes_simultaneously() throws Exec // Accepts options - @Test - public void should_Cache_failed_fetches() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -471,11 +615,12 @@ public void should_Cache_failed_fetches() { assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); } - @Test - public void should_NOT_Cache_failed_fetches_if_told_not_too() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactory factory) { + DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false).build(); List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(options, loadCalls); + DataLoader errorLoader = factory.idLoaderAllExceptions(options, loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -497,10 +642,11 @@ public void should_NOT_Cache_failed_fetches_if_told_not_too() { // Accepts object key in custom cacheKey function - @Test - public void should_Handle_priming_the_cache_with_an_error() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Handle_priming_the_cache_with_an_error(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime(1, new IllegalStateException("Error")); @@ -513,10 +659,11 @@ public void should_Handle_priming_the_cache_with_an_error() { assertThat(loadCalls, equalTo(emptyList())); } - @Test - public void should_Clear_values_from_cache_after_errors() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Clear_values_from_cache_after_errors(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); future1.handle((value, t) -> { @@ -548,10 +695,11 @@ public void should_Clear_values_from_cache_after_errors() { assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); } - @Test - public void should_Propagate_error_to_all_loads() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Propagate_error_to_all_loads(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); CompletableFuture future2 = errorLoader.load(2); @@ -571,10 +719,11 @@ public void should_Propagate_error_to_all_loads() { assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } - @Test - public void should_Accept_objects_as_keys() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); Object keyA = new Object(); Object keyB = new Object(); @@ -612,11 +761,12 @@ public void should_Accept_objects_as_keys() { assertThat(loadCalls.get(1).toArray()[0], equalTo(keyA)); } - @Test - public void should_Disable_caching() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Disable_caching(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -649,11 +799,12 @@ public void should_Disable_caching() throws ExecutionException, InterruptedExcep asList("A", "C"), asList("A", "B", "C")))); } - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -664,14 +815,19 @@ public void should_work_with_duplicate_keys_when_caching_disabled() throws Execu assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); + if (factory.unwrap() instanceof MappedDataLoaderFactory || factory.unwrap() instanceof MappedPublisherDataLoaderFactory) { + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + } else { + assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); + } } - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(true), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(true).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -687,17 +843,18 @@ public void should_work_with_duplicate_keys_when_caching_enabled() throws Execut // It is resilient to job queue ordering - @Test - public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); identityLoader.dispatch(); await().until(() -> future1.isDone() && future2.isDone()); @@ -708,22 +865,23 @@ public void should_Accept_objects_with_a_complex_key() throws ExecutionException // Helper methods - @Test - public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); - CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future1 = identityLoader.load(key1); identityLoader.dispatch(); await().until(future1::isDone); identityLoader.clear(key2); // clear equivalent object key - CompletableFuture future2 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key1); identityLoader.dispatch(); await().until(future2::isDone); @@ -732,33 +890,35 @@ public void should_Clear_objects_with_complex_key() throws ExecutionException, I assertThat(future2.get(), equalTo(key1)); } - @Test - public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); // Fetches as expected - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); identityLoader.dispatch(); await().until(() -> future1.isDone() && future2.isDone()); assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); assertThat(loadCalls.size(), equalTo(1)); assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key2)); } - @Test - public void should_Allow_priming_the_cache_with_an_object_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); @@ -775,12 +935,13 @@ public void should_Allow_priming_the_cache_with_an_object_key() throws Execution assertThat(future2.get(), equalTo(key1)); } - @Test - public void should_Accept_a_custom_cache_map_implementation() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { CustomCacheMap customMap = new CustomCacheMap(); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheMap(customMap); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setCacheMap(customMap).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -826,11 +987,27 @@ public void should_Accept_a_custom_cache_map_implementation() throws ExecutionEx assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); } - @Test - public void batching_disabled_should_dispatch_immediately() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory factory) { + CacheMap cache = new ThrowingCacheMap(); + DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache).build(); + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); + + assertThat(identityLoader.getIfPresent("a"), equalTo(Optional.empty())); + + CompletableFuture future = identityLoader.load("a"); + identityLoader.dispatch(); + assertThat(future.join(), equalTo("a")); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setBatchingEnabled(false).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); CompletableFuture fb = identityLoader.load("B"); @@ -854,11 +1031,12 @@ public void batching_disabled_should_dispatch_immediately() { } - @Test - public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); CompletableFuture fb = identityLoader.load("B"); @@ -885,10 +1063,11 @@ public void batching_disabled_and_caching_disabled_should_dispatch_immediately_a } - @Test - public void batches_multiple_requests_with_max_batch_size() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(2), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(2).build(), loadCalls); CompletableFuture f1 = identityLoader.load(1); CompletableFuture f2 = identityLoader.load(2); @@ -896,7 +1075,7 @@ public void batches_multiple_requests_with_max_batch_size() { identityLoader.dispatch(); - CompletableFuture.allOf(f1, f2, f3).join(); + allOf(f1, f2, f3).join(); assertThat(f1.join(), equalTo(1)); assertThat(f2.join(), equalTo(2)); @@ -906,10 +1085,11 @@ public void batches_multiple_requests_with_max_batch_size() { } - @Test - public void can_split_max_batch_sizes_correctly() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(5), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(5).build(), loadCalls); for (int i = 0; i < 21; i++) { identityLoader.load(i); @@ -928,22 +1108,23 @@ public void can_split_max_batch_sizes_correctly() { } - @Test - public void should_Batch_loads_occurring_within_futures() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(newDefaultOptions(), loadCalls); Supplier nullValue = () -> null; AtomicBoolean v4Called = new AtomicBoolean(); - CompletableFuture.supplyAsync(nullValue).thenAccept(v1 -> { + supplyAsync(nullValue).thenAccept(v1 -> { identityLoader.load("a"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v2 -> { + supplyAsync(nullValue).thenAccept(v2 -> { identityLoader.load("b"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v3 -> { + supplyAsync(nullValue).thenAccept(v3 -> { identityLoader.load("c"); - CompletableFuture.supplyAsync(nullValue).thenAccept( + supplyAsync(nullValue).thenAccept( v4 -> { identityLoader.load("d"); v4Called.set(true); @@ -960,12 +1141,101 @@ public void should_Batch_loads_occurring_within_futures() { singletonList(asList("a", "b", "c", "d")))); } + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_blowup_after_N_keys(TestDataLoaderFactory factory) { + if (!(factory instanceof TestReactiveDataLoaderFactory)) { + return; + } + // + // if we blow up after emitting N keys, the N keys should work but the rest of the keys + // should be exceptional + DataLoader identityLoader = ((TestReactiveDataLoaderFactory) factory).idLoaderBlowsUpsAfterN(3, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load(1); + CompletableFuture cf2 = identityLoader.load(2); + CompletableFuture cf3 = identityLoader.load(3); + CompletableFuture cf4 = identityLoader.load(4); + CompletableFuture cf5 = identityLoader.load(5); + identityLoader.dispatch(); + await().until(cf5::isDone); + + assertThat(cf1.join(), equalTo(1)); + assertThat(cf2.join(), equalTo(2)); + assertThat(cf3.join(), equalTo(3)); + assertThat(cf4.isCompletedExceptionally(), is(true)); + assertThat(cf5.isCompletedExceptionally(), is(true)); + + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void when_values_size_are_less_then_key_size(TestDataLoaderFactory factory) { + // + // what happens if we want 4 values but are only given 2 back say + // + DataLoader identityLoader = factory.onlyReturnsNValues(2, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load("A"); + CompletableFuture cf2 = identityLoader.load("B"); + CompletableFuture cf3 = identityLoader.load("C"); + CompletableFuture cf4 = identityLoader.load("D"); + identityLoader.dispatch(); + + await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); + + if (factory.unwrap() instanceof ListDataLoaderFactory) { + assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else if (factory.unwrap() instanceof PublisherDataLoaderFactory) { + // some have completed progressively but the other never did + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else { + // with the maps it's ok to have fewer results + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cf3.join(), equalTo(null)); + assertThat(cf4.join(), equalTo(null)); + } + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void when_values_size_are_more_then_key_size(TestDataLoaderFactory factory) { + // + // what happens if we want 4 values but only given 6 back say + // + DataLoader identityLoader = factory.idLoaderReturnsTooMany(2, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load("A"); + CompletableFuture cf2 = identityLoader.load("B"); + CompletableFuture cf3 = identityLoader.load("C"); + CompletableFuture cf4 = identityLoader.load("D"); + identityLoader.dispatch(); + await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); + + + if (factory.unwrap() instanceof ListDataLoaderFactory) { + assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else { + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cf3.join(), equalTo("C")); + assertThat(cf4.join(), equalTo("D")); + } + } + @Test public void can_call_a_loader_from_a_loader() throws Exception { List> deepLoadCalls = new ArrayList<>(); DataLoader deepLoader = newDataLoader(keys -> { deepLoadCalls.add(keys); - return CompletableFuture.completedFuture(keys); + return completedFuture(keys); }); List> aLoadCalls = new ArrayList<>(); @@ -985,7 +1255,7 @@ public void can_call_a_loader_from_a_loader() throws Exception { CompletableFuture b1 = bLoader.load("B1"); CompletableFuture b2 = bLoader.load("B2"); - CompletableFuture.allOf( + allOf( aLoader.dispatch(), deepLoader.dispatch(), bLoader.dispatch(), @@ -1011,11 +1281,10 @@ public void can_call_a_loader_from_a_loader() throws Exception { public void should_allow_composition_of_data_loader_calls() { UserManager userManager = new UserManager(); - BatchLoader userBatchLoader = userIds -> CompletableFuture - .supplyAsync(() -> userIds - .stream() - .map(userManager::loadUserById) - .collect(Collectors.toList())); + BatchLoader userBatchLoader = userIds -> supplyAsync(() -> userIds + .stream() + .map(userManager::loadUserById) + .collect(Collectors.toList())); DataLoader userLoader = newDataLoader(userBatchLoader); AtomicBoolean gandalfCalled = new AtomicBoolean(false); @@ -1051,56 +1320,12 @@ private static CacheKey getJsonObjectCacheMapFn() { .collect(Collectors.joining()); } - private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - @SuppressWarnings("unchecked") - List values = keys.stream() - .map(k -> (V) k) - .collect(Collectors.toList()); - return CompletableFuture.completedFuture(values); - }, options); - } - - private static DataLoader idLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - return TestKit.futureError(); - }, options); - } - - private static DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); - return CompletableFuture.completedFuture(errors); - }, options); - } + private static class ThrowingCacheMap extends CustomCacheMap { - private static DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - List errors = new ArrayList<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errors.add(key); - } else { - errors.add(new IllegalStateException("Error")); - } - } - return CompletableFuture.completedFuture(errors); - }, options); - } - - - private BatchLoader keysAsValues() { - return CompletableFuture::completedFuture; + @Override + public CompletableFuture get(String key) { + throw new RuntimeException("Cache implementation failed."); + } } - } diff --git a/src/test/java/org/dataloader/DataLoaderTimeTest.java b/src/test/java/org/dataloader/DataLoaderTimeTest.java index ee73d85..b4d645c 100644 --- a/src/test/java/org/dataloader/DataLoaderTimeTest.java +++ b/src/test/java/org/dataloader/DataLoaderTimeTest.java @@ -1,13 +1,13 @@ package org.dataloader; import org.dataloader.fixtures.TestingClock; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.time.Instant; import static org.dataloader.fixtures.TestKit.keysAsValues; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; @SuppressWarnings("UnusedReturnValue") public class DataLoaderTimeTest { diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 38cfe77..24d111b 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -4,10 +4,13 @@ import com.github.benmanes.caffeine.cache.Caffeine; import org.dataloader.fixtures.CaffeineValueCache; import org.dataloader.fixtures.CustomValueCache; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; import org.dataloader.impl.DataLoaderAssertionException; -import org.junit.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -18,23 +21,23 @@ import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.idLoader; import static org.dataloader.fixtures.TestKit.snooze; import static org.dataloader.fixtures.TestKit.sort; import static org.dataloader.impl.CompletableFutureKit.failedFuture; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class DataLoaderValueCacheTest { - @Test - public void test_by_default_we_have_no_value_caching() { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions(); - DataLoader identityLoader = idLoader(options, loadCalls); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory factory) { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -64,12 +67,13 @@ public void test_by_default_we_have_no_value_caching() { assertThat(loadCalls, equalTo(emptyList())); } - @Test - public void should_accept_a_remote_value_store_for_caching() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache(); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -108,8 +112,9 @@ public void should_accept_a_remote_value_store_for_caching() { assertArrayEquals(customValueCache.store.keySet().toArray(), emptyList().toArray()); } - @Test - public void can_use_caffeine_for_caching() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { // // Mostly to prove that some other CACHE library could be used // as the backing value cache. Not really Caffeine specific. @@ -121,9 +126,9 @@ public void can_use_caffeine_for_caching() { ValueCache caffeineValueCache = new CaffeineValueCache(caffeineCache); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(caffeineValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(caffeineValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -148,8 +153,9 @@ public void can_use_caffeine_for_caching() { assertArrayEquals(caffeineCache.asMap().keySet().toArray(), asList("a", "b", "c").toArray()); } - @Test - public void will_invoke_loader_if_CACHE_GET_call_throws_exception() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void will_invoke_loader_if_CACHE_GET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -163,9 +169,9 @@ public CompletableFuture get(String key) { customValueCache.set("a", "Not From Cache"); customValueCache.set("b", "From Cache"); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -174,12 +180,13 @@ public CompletableFuture get(String key) { assertThat(fA.join(), equalTo("a")); assertThat(fB.join(), equalTo("From Cache")); - // a was not in cache (according to get) and hence needed to be loaded + // "a" was not in cache (according to get) and hence needed to be loaded assertThat(loadCalls, equalTo(singletonList(singletonList("a")))); } - @Test - public void will_still_work_if_CACHE_SET_call_throws_exception() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void will_still_work_if_CACHE_SET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override public CompletableFuture set(String key, Object value) { @@ -190,9 +197,9 @@ public CompletableFuture set(String key, Object value) { } }; - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -201,13 +208,14 @@ public CompletableFuture set(String key, Object value) { assertThat(fA.join(), equalTo("a")); assertThat(fB.join(), equalTo("b")); - // a was not in cache (according to get) and hence needed to be loaded + // "a" was not in cache (according to get) and hence needed to be loaded assertThat(loadCalls, equalTo(singletonList(asList("a", "b")))); assertArrayEquals(customValueCache.store.keySet().toArray(), singletonList("b").toArray()); } - @Test - public void caching_can_take_some_time_complete() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void caching_can_take_some_time_complete(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -228,9 +236,9 @@ public CompletableFuture get(String key) { }; - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -247,8 +255,9 @@ public CompletableFuture get(String key) { assertThat(loadCalls, equalTo(singletonList(asList("missC", "missD")))); } - @Test - public void batch_caching_works_as_expected() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void batch_caching_works_as_expected(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -269,9 +278,9 @@ public CompletableFuture>> getValues(List keys) { }; - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -288,13 +297,14 @@ public CompletableFuture>> getValues(List keys) { assertThat(loadCalls, equalTo(singletonList(asList("missC", "missD")))); List values = new ArrayList<>(customValueCache.asMap().values()); - // it will only set back in values that are missed - it wont set in values that successfully + // it will only set back in values that are missed - it won't set in values that successfully // came out of the cache assertThat(values, equalTo(asList("missC", "missD"))); } - @Test - public void assertions_will_be_thrown_if_the_cache_does_not_follow_contract() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void assertions_will_be_thrown_if_the_cache_does_not_follow_contract(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -312,9 +322,9 @@ public CompletableFuture>> getValues(List keys) { } }; - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -335,8 +345,9 @@ private boolean isAssertionException(CompletableFuture fA) { } - @Test - public void if_caching_is_off_its_never_hit() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void if_caching_is_off_its_never_hit(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -347,9 +358,9 @@ public CompletableFuture get(String key) { } }; - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(false).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -368,8 +379,9 @@ public CompletableFuture get(String key) { assertTrue(customValueCache.asMap().isEmpty()); } - @Test - public void if_everything_is_cached_no_batching_happens() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void if_everything_is_cached_no_batching_happens(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -390,9 +402,9 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("b", "cachedB"); customValueCache.asMap().put("c", "cachedC"); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -410,8 +422,9 @@ public CompletableFuture> setValues(List keys, List } - @Test - public void if_batching_is_off_it_still_can_cache() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void if_batching_is_off_it_still_can_cache(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -430,9 +443,9 @@ public CompletableFuture> setValues(List keys, List }; customValueCache.asMap().put("a", "cachedA"); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).setBatchingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).setBatchingEnabled(false).build(); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); diff --git a/src/test/java/org/dataloader/DataLoaderWithTryTest.java b/src/test/java/org/dataloader/DataLoaderWithTryTest.java index e9e8538..fda7bd4 100644 --- a/src/test/java/org/dataloader/DataLoaderWithTryTest.java +++ b/src/test/java/org/dataloader/DataLoaderWithTryTest.java @@ -1,7 +1,7 @@ package org.dataloader; import org.hamcrest.Matchers; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.HashMap; @@ -12,9 +12,9 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.dataloader.DataLoaderFactory.*; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertThat; public class DataLoaderWithTryTest { diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java new file mode 100644 index 0000000..8849752 --- /dev/null +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -0,0 +1,80 @@ +package org.dataloader; + +import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +/** + * There are WAY more tests via the {@link DelegatingDataLoaderFactory} + * parameterized tests. All the basic {@link DataLoader} tests pass when wrapped in a {@link DelegatingDataLoader} + */ +public class DelegatingDataLoaderTest { + + @Test + void canUnwrapDataLoaders() { + DataLoader rawLoader = TestKit.idLoader(); + DataLoader delegateLoader = new DelegatingDataLoader<>(rawLoader); + + assertThat(DelegatingDataLoader.unwrap(rawLoader), is(rawLoader)); + assertThat(DelegatingDataLoader.unwrap(delegateLoader), is(rawLoader)); + } + + @Test + void canCreateAClassOk() { + DataLoader rawLoader = TestKit.idLoader(); + DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) { + @Override + public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) { + CompletableFuture cf = super.load(key, keyContext); + return cf.thenApply(v -> "|" + v + "|"); + } + }; + + assertThat(delegatingDataLoader.getDelegate(), is(rawLoader)); + + + CompletableFuture cfA = delegatingDataLoader.load("A"); + CompletableFuture cfB = delegatingDataLoader.load("B"); + CompletableFuture> cfCD = delegatingDataLoader.loadMany(List.of("C", "D")); + + CompletableFuture> dispatch = delegatingDataLoader.dispatch(); + + await().until(dispatch::isDone); + + assertThat(cfA.join(), equalTo("|A|")); + assertThat(cfB.join(), equalTo("|B|")); + assertThat(cfCD.join(), equalTo(List.of("|C|", "|D|"))); + + assertThat(delegatingDataLoader.getIfPresent("A").isEmpty(), equalTo(false)); + assertThat(delegatingDataLoader.getIfPresent("X").isEmpty(), equalTo(true)); + + assertThat(delegatingDataLoader.getIfCompleted("A").isEmpty(), equalTo(false)); + assertThat(delegatingDataLoader.getIfCompleted("X").isEmpty(), equalTo(true)); + } + + @Test + void can_delegate_simple_properties() { + DataLoaderOptions options = DataLoaderOptions.newOptions().build(); + BatchLoader loadFunction = CompletableFuture::completedFuture; + + DataLoader rawLoader = DataLoaderFactory.newDataLoader("name", loadFunction, options); + DelegatingDataLoader delegate = new DelegatingDataLoader<>(rawLoader); + + assertNotNull(delegate.getName()); + assertThat(delegate.getName(),equalTo("name")); + assertThat(delegate.getOptions(),equalTo(options)); + assertThat(delegate.getBatchLoadFunction(),equalTo(loadFunction)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/TryTest.java b/src/test/java/org/dataloader/TryTest.java index 1fdd286..1b237e2 100644 --- a/src/test/java/org/dataloader/TryTest.java +++ b/src/test/java/org/dataloader/TryTest.java @@ -1,7 +1,7 @@ package org.dataloader; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -9,13 +9,12 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -@SuppressWarnings("ConstantConditions") public class TryTest { interface RunThatCanThrow { @@ -30,9 +29,10 @@ private void expectThrowable(RunThatCanThrow runnable, Class sTry, String expectedString) { assertThat(sTry.isSuccess(), equalTo(false)); assertThat(sTry.isFailure(), equalTo(true)); @@ -51,21 +51,21 @@ private void assertSuccess(Try sTry, String expectedStr) { } @Test - public void tryFailed() throws Exception { + public void tryFailed() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); assertFailure(sTry, "Goodbye Cruel World"); } @Test - public void trySucceeded() throws Exception { + public void trySucceeded() { Try sTry = Try.succeeded("Hello World"); assertSuccess(sTry, "Hello World"); } @Test - public void tryCallable() throws Exception { + public void tryCallable() { Try sTry = Try.tryCall(() -> "Hello World"); assertSuccess(sTry, "Hello World"); @@ -78,7 +78,7 @@ public void tryCallable() throws Exception { } @Test - public void triedStage() throws Exception { + public void triedStage() { CompletionStage> sTry = Try.tryStage(CompletableFuture.completedFuture("Hello World")); sTry.thenAccept(stageTry -> assertSuccess(stageTry, "Hello World")); @@ -93,7 +93,7 @@ public void triedStage() throws Exception { } @Test - public void map() throws Exception { + public void map() { Try iTry = Try.succeeded(666); Try sTry = iTry.map(Object::toString); @@ -106,7 +106,7 @@ public void map() throws Exception { } @Test - public void flatMap() throws Exception { + public void flatMap() { Function> intToStringFunc = i -> Try.succeeded(i.toString()); Try iTry = Try.succeeded(666); @@ -123,7 +123,7 @@ public void flatMap() throws Exception { } @Test - public void toOptional() throws Exception { + public void toOptional() { Try iTry = Try.succeeded(666); Optional optional = iTry.toOptional(); assertThat(optional.isPresent(), equalTo(true)); @@ -135,7 +135,7 @@ public void toOptional() throws Exception { } @Test - public void orElse() throws Exception { + public void orElse() { Try sTry = Try.tryCall(() -> "Hello World"); String result = sTry.orElse("other"); @@ -147,7 +147,7 @@ public void orElse() throws Exception { } @Test - public void orElseGet() throws Exception { + public void orElseGet() { Try sTry = Try.tryCall(() -> "Hello World"); String result = sTry.orElseGet(() -> "other"); @@ -159,7 +159,7 @@ public void orElseGet() throws Exception { } @Test - public void reThrow() throws Exception { + public void reThrow() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); expectThrowable(sTry::reThrow, RuntimeException.class); @@ -169,7 +169,7 @@ public void reThrow() throws Exception { } @Test - public void forEach() throws Exception { + public void forEach() { AtomicReference sRef = new AtomicReference<>(); Try sTry = Try.tryCall(() -> "Hello World"); sTry.forEach(sRef::set); @@ -183,7 +183,7 @@ public void forEach() throws Exception { } @Test - public void recover() throws Exception { + public void recover() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); sTry = sTry.recover(t -> "Hello World"); diff --git a/src/test/java/org/dataloader/ValueCacheOptionsTest.java b/src/test/java/org/dataloader/ValueCacheOptionsTest.java new file mode 100644 index 0000000..469e291 --- /dev/null +++ b/src/test/java/org/dataloader/ValueCacheOptionsTest.java @@ -0,0 +1,19 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class ValueCacheOptionsTest { + + @Test + void saneDefaults() { + ValueCacheOptions newOptions = ValueCacheOptions.newOptions(); + assertThat(newOptions.isCompleteValueAfterCacheSet(), equalTo(false)); + + ValueCacheOptions differentOptions = newOptions.setCompleteValueAfterCacheSet(true); + assertThat(differentOptions.isCompleteValueAfterCacheSet(), equalTo(true)); + assertThat(differentOptions == newOptions, equalTo(false)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/fixtures/Stopwatch.java b/src/test/java/org/dataloader/fixtures/Stopwatch.java new file mode 100644 index 0000000..c815a8b --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/Stopwatch.java @@ -0,0 +1,57 @@ +package org.dataloader.fixtures; + +import java.time.Duration; + +public class Stopwatch { + + public static Stopwatch stopwatchStarted() { + return new Stopwatch().start(); + } + + public static Stopwatch stopwatchUnStarted() { + return new Stopwatch(); + } + + private long started = -1; + private long stopped = -1; + + public Stopwatch start() { + synchronized (this) { + if (started != -1) { + throw new IllegalStateException("You have started it before"); + } + started = System.currentTimeMillis(); + } + return this; + } + + private Stopwatch() { + } + + public long elapsed() { + synchronized (this) { + if (started == -1) { + throw new IllegalStateException("You haven't started it"); + } + if (stopped == -1) { + return System.currentTimeMillis() - started; + } else { + return stopped - started; + } + } + } + + public Duration duration() { + return Duration.ofMillis(elapsed()); + } + + public Duration stop() { + synchronized (this) { + if (started != -1) { + throw new IllegalStateException("You have started it"); + } + stopped = System.currentTimeMillis(); + return duration(); + } + } +} diff --git a/src/test/java/org/dataloader/fixtures/TestKit.java b/src/test/java/org/dataloader/fixtures/TestKit.java index 5c87148..e6ba319 100644 --- a/src/test/java/org/dataloader/fixtures/TestKit.java +++ b/src/test/java/org/dataloader/fixtures/TestKit.java @@ -1,13 +1,21 @@ package org.dataloader.fixtures; import org.dataloader.BatchLoader; +import org.dataloader.BatchLoaderWithContext; import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; +import org.dataloader.MappedBatchLoader; +import org.dataloader.MappedBatchLoaderWithContext; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import static java.util.stream.Collectors.toList; @@ -19,6 +27,27 @@ public static BatchLoader keysAsValues() { return CompletableFuture::completedFuture; } + public static BatchLoaderWithContext keysAsValuesWithContext() { + return (keys, env) -> CompletableFuture.completedFuture(keys); + } + + public static MappedBatchLoader keysAsMapOfValues() { + return TestKit::mapOfKeys; + } + + public static MappedBatchLoaderWithContext keysAsMapOfValuesWithContext() { + return (keys, env) -> mapOfKeys(keys); + } + + private static CompletableFuture> mapOfKeys(Set keys) { + Map map = new HashMap<>(); + for (K key : keys) { + //noinspection unchecked + map.put(key, (V) key); + } + return CompletableFuture.completedFuture(map); + } + public static BatchLoader keysAsValues(List> loadCalls) { return keys -> { List ks = new ArrayList<>(keys); @@ -35,14 +64,18 @@ public static DataLoader idLoader() { return idLoader(null, new ArrayList<>()); } - public static DataLoader idLoader(List> loadCalls) { - return idLoader(null, loadCalls); + public static DataLoader idLoader(String name) { + return idLoader(name, null, new ArrayList<>()); } public static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { return DataLoaderFactory.newDataLoader(keysAsValues(loadCalls), options); } + public static DataLoader idLoader(String name, DataLoaderOptions options, List> loadCalls) { + return DataLoaderFactory.newDataLoader(name, keysAsValues(loadCalls), options); + } + public static Collection listFrom(int i, int max) { List ints = new ArrayList<>(); for (int j = i; j < max; j++) { @@ -55,7 +88,7 @@ public static CompletableFuture futureError() { return failedFuture(new IllegalStateException("Error")); } - public static void snooze(int millis) { + public static void snooze(long millis) { try { Thread.sleep(millis); } catch (InterruptedException e) { @@ -67,4 +100,22 @@ public static void snooze(int millis) { public static List sort(Collection collection) { return collection.stream().sorted().collect(toList()); } + + @SafeVarargs + public static Set asSet(T... elements) { + return new LinkedHashSet<>(Arrays.asList(elements)); + } + + public static Set asSet(Collection elements) { + return new LinkedHashSet<>(elements); + } + + public static boolean areAllDone(CompletableFuture... cfs) { + for (CompletableFuture cf : cfs) { + if (!cf.isDone()) { + return false; + } + } + return true; + } } diff --git a/src/test/java/org/dataloader/fixtures/UserManager.java b/src/test/java/org/dataloader/fixtures/UserManager.java index 24fee0d..1d2ff1f 100644 --- a/src/test/java/org/dataloader/fixtures/UserManager.java +++ b/src/test/java/org/dataloader/fixtures/UserManager.java @@ -1,5 +1,8 @@ package org.dataloader.fixtures; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -52,6 +55,14 @@ public List loadUsersById(List userIds) { return userIds.stream().map(this::loadUserById).collect(Collectors.toList()); } + public Publisher streamUsersById(List userIds) { + return Flux.fromIterable(loadUsersById(userIds)); + } + + public Publisher> streamUsersById(Set userIds) { + return Flux.fromIterable(loadMapOfUsersByIds(null, userIds).entrySet()); + } + public Map loadMapOfUsersByIds(SecurityCtx callCtx, Set userIds) { Map map = new HashMap<>(); userIds.forEach(userId -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java new file mode 100644 index 0000000..0cbd3f3 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java @@ -0,0 +1,71 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DelegatingDataLoader; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class DelegatingDataLoaderFactory implements TestDataLoaderFactory { + // its delegates all the way down to the turtles + private final TestDataLoaderFactory delegateFactory; + + public DelegatingDataLoaderFactory(TestDataLoaderFactory delegateFactory) { + this.delegateFactory = delegateFactory; + } + + @Override + public String toString() { + return "DelegatingDataLoaderFactory{" + + "delegateFactory=" + delegateFactory + + '}'; + } + + @Override + public TestDataLoaderFactory unwrap() { + return delegateFactory.unwrap(); + } + + private DataLoader mkDelegateDataLoader(DataLoader dataLoader) { + return new DelegatingDataLoader<>(dataLoader); + } + + @Override + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoader(options, loadCalls)); + } + + @Override + public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return mkDelegateDataLoader(delegateFactory.idLoaderDelayed(options, loadCalls, delay)); + } + + @Override + public DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderBlowsUps(options, loadCalls)); + } + + @Override + public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderAllExceptions(options, loadCalls)); + } + + @Override + public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderOddEvenExceptions(options, loadCalls)); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return mkDelegateDataLoader(delegateFactory.onlyReturnsNValues(N, options, loadCalls)); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderReturnsTooMany(howManyMore, options, loadCalls)); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java new file mode 100644 index 0000000..0644d3c --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java @@ -0,0 +1,90 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.fixtures.TestKit; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.dataloader.DataLoaderFactory.newDataLoader; + +public class ListDataLoaderFactory implements TestDataLoaderFactory { + @Override + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys); + }, options); + } + + @Override + public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return newDataLoader(keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + return keys; + })); + } + + @Override + public DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return TestKit.futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); + return completedFuture(errors); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(key); + } else { + errors.add(new IllegalStateException("Error")); + } + } + return completedFuture(errors); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys.subList(0, N)); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + return completedFuture(l); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java new file mode 100644 index 0000000..e7c47ec --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java @@ -0,0 +1,111 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.fixtures.TestKit; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.dataloader.fixtures.TestKit.futureError; + +public class MappedDataLoaderFactory implements TestDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + return completedFuture(map); + }, options); + } + + @Override + public DataLoader idLoaderDelayed( + DataLoaderOptions options, List> loadCalls, Duration delay) { + return newMappedDataLoader(keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + return map; + })); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + return futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + Map errorByKey = new HashMap<>(); + keys.forEach(k -> errorByKey.put(k, new IllegalStateException("Error"))); + return completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, key); + } else { + errorByKey.put(key, new IllegalStateException("Error")); + } + } + return completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map collect = List.copyOf(keys).subList(0, N).stream().collect(Collectors.toMap( + k -> k, v -> v + )); + return completedFuture(collect); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Map collect = l.stream().collect(Collectors.toMap( + k -> k, v -> v + )); + return completedFuture(collect); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java new file mode 100644 index 0000000..fa920cf --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java @@ -0,0 +1,126 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.Try; +import org.dataloader.fixtures.TestKit; +import reactor.core.publisher.Flux; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; + +public class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderDelayed( + DataLoaderOptions options, List> loadCalls, Duration delay) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + CompletableFuture.runAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream>> failures = keys.stream().map(k -> Map.entry(k, Try.failed(new IllegalStateException("Error")))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + Map> errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, Try.succeeded(key)); + } else { + errorByKey.put(key, Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errorByKey.entrySet()).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.stream().limit(N).collect(toList()); + Flux> subFlux = Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.stream().limit(N).collect(toList()); + Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Flux.fromIterable(l).map(k -> Map.entry(k, k)) + .subscribe(subscriber); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java new file mode 100644 index 0000000..2049719 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java @@ -0,0 +1,115 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.Try; +import org.dataloader.fixtures.TestKit; +import reactor.core.publisher.Flux; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; + +import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; + +public class PublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return newPublisherDataLoader((keys, subscriber) -> { + CompletableFuture.runAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List> errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(Try.succeeded(key)); + } else { + errors.add(Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errors).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux subFlux = Flux.fromIterable(nKeys); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux.fromIterable(nKeys) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Flux.fromIterable(l) + .subscribe(subscriber); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java new file mode 100644 index 0000000..48678c4 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java @@ -0,0 +1,25 @@ +package org.dataloader.fixtures.parameterized; + +import org.junit.jupiter.api.Named; +import org.junit.jupiter.params.provider.Arguments; + +import java.util.stream.Stream; + +@SuppressWarnings("unused") +public class TestDataLoaderFactories { + + public static Stream get() { + return Stream.of( + Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), + Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), + Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())), + + // runs all the above via a DelegateDataLoader + Arguments.of(Named.of("Delegate List DataLoader", new DelegatingDataLoaderFactory(new ListDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Mapped DataLoader", new DelegatingDataLoaderFactory(new MappedDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Publisher DataLoader", new DelegatingDataLoaderFactory(new PublisherDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Mapped Publisher DataLoader", new DelegatingDataLoaderFactory(new MappedPublisherDataLoaderFactory()))) + ); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java new file mode 100644 index 0000000..789b136 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -0,0 +1,46 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public interface TestDataLoaderFactory { + DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay); + + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); + + DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls); + + DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls); + + // Convenience methods + + default DataLoader idLoader(DataLoaderOptions options) { + return idLoader(options, new ArrayList<>()); + } + + default DataLoader idLoader(List> calls) { + return idLoader(null, calls); + } + default DataLoader idLoader() { + return idLoader(null, new ArrayList<>()); + } + + default DataLoader idLoaderDelayed(Duration delay) { + return idLoaderDelayed(null, new ArrayList<>(), delay); + } + + default TestDataLoaderFactory unwrap() { + return this; + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java new file mode 100644 index 0000000..d45932c --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java @@ -0,0 +1,11 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; + +import java.util.Collection; +import java.util.List; + +public interface TestReactiveDataLoaderFactory { + DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls); +} diff --git a/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java b/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java index cbf8cc8..6073319 100644 --- a/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java +++ b/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java @@ -1,6 +1,6 @@ package org.dataloader.impl; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; @@ -12,12 +12,12 @@ import static java.util.Arrays.asList; import static java.util.concurrent.CompletableFuture.supplyAsync; import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertThat; public class PromisedValuesImplTest { @@ -186,7 +186,7 @@ public void exceptions_are_captured_and_reported() throws Exception { @Test public void type_generics_compile_as_expected() throws Exception { - PromisedValues pvList = PromisedValues.allOf(Collections.singletonList(new CompletableFuture())); + PromisedValues pvList = PromisedValues.allOf(Collections.singletonList(new CompletableFuture<>())); PromisedValues pvList2 = PromisedValues.allOf(Collections.>singletonList(new CompletableFuture<>())); assertThat(pvList, notNullValue()); diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java new file mode 100644 index 0000000..b11bc27 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java @@ -0,0 +1,83 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +class CapturingInstrumentation implements DataLoaderInstrumentation { + protected String name; + protected List methods = new ArrayList<>(); + + public CapturingInstrumentation(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public List methods() { + return methods; + } + + public List notLoads() { + return methods.stream().filter(method -> !method.contains("beginLoad")).collect(Collectors.toList()); + } + + public List onlyLoads() { + return methods.stream().filter(method -> method.contains("beginLoad")).collect(Collectors.toList()); + } + + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + methods.add(name + "_beginLoad" +"_k:" + key); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginLoad_onDispatched"+"_k:" + key); + } + + @Override + public void onCompleted(Object result, Throwable t) { + methods.add(name + "_beginLoad_onCompleted"+"_k:" + key); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginDispatch_onDispatched"); + } + + @Override + public void onCompleted(DispatchResult result, Throwable t) { + methods.add(name + "_beginDispatch_onCompleted"); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginBatchLoader_onDispatched"); + } + + @Override + public void onCompleted(List result, Throwable t) { + methods.add(name + "_beginBatchLoader_onCompleted"); + } + }; + } +} diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java new file mode 100644 index 0000000..4d2f0f4 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java @@ -0,0 +1,32 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.List; + +class CapturingInstrumentationReturnsNull extends CapturingInstrumentation { + + public CapturingInstrumentationReturnsNull(String name) { + super(name); + } + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + methods.add(name + "_beginLoad" +"_k:" + key); + return null; + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return null; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return null; + } +} diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java new file mode 100644 index 0000000..28cef70 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -0,0 +1,130 @@ +package org.dataloader.instrumentation; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.dataloader.DataLoaderOptions; +import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderOptions.newOptions; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class ChainedDataLoaderInstrumentationTest { + + CapturingInstrumentation capturingA; + CapturingInstrumentation capturingB; + CapturingInstrumentation capturingButReturnsNull; + + + @BeforeEach + void setUp() { + capturingA = new CapturingInstrumentation("A"); + capturingB = new CapturingInstrumentation("B"); + capturingButReturnsNull = new CapturingInstrumentationReturnsNull("NULL"); + } + + @Test + void canChainTogetherZeroInstrumentation() { + // just to prove its useless but harmless + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation(); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); + + DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); + + dl.load("A"); + dl.load("B"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(dispatch.join(), equalTo(List.of("A", "B"))); + } + + @Test + void canChainTogetherOneInstrumentation() { + CapturingInstrumentation capturingA = new CapturingInstrumentation("A"); + + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); + + DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); + + dl.load("X"); + dl.load("Y"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + + assertThat(capturingA.notLoads(), equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + + assertThat(capturingA.onlyLoads(), equalTo(List.of( + "A_beginLoad_k:X", "A_beginLoad_onDispatched_k:X", "A_beginLoad_k:Y", "A_beginLoad_onDispatched_k:Y", + "A_beginLoad_onCompleted_k:X", "A_beginLoad_onCompleted_k:Y" + ))); + } + + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDataLoaderFactory factory) { + + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA) + .add(capturingB) + .add(capturingButReturnsNull); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); + + DataLoader dl = factory.idLoader(options); + + dl.load("X"); + dl.load("Y"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + + // + // A_beginBatchLoader happens before A_beginDispatch_onDispatched because these are sync + // and no async - a batch scheduler or async batch loader would change that + // + assertThat(capturingA.notLoads(), equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + + assertThat(capturingA.onlyLoads(), equalTo(List.of( + "A_beginLoad_k:X", "A_beginLoad_onDispatched_k:X", "A_beginLoad_k:Y", "A_beginLoad_onDispatched_k:Y", + "A_beginLoad_onCompleted_k:X", "A_beginLoad_onCompleted_k:Y" + ))); + + assertThat(capturingB.notLoads(), equalTo(List.of("B_beginDispatch", + "B_beginBatchLoader", "B_beginBatchLoader_onDispatched", "B_beginBatchLoader_onCompleted", + "B_beginDispatch_onDispatched", "B_beginDispatch_onCompleted"))); + + // it returned null on all its contexts - nothing to call back on + assertThat(capturingButReturnsNull.notLoads(), equalTo(List.of("NULL_beginDispatch", "NULL_beginBatchLoader"))); + } + + @Test + void addition_works() { + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA).prepend(capturingB).addAll(List.of(capturingButReturnsNull)); + + assertThat(chainedItn.getInstrumentations(), equalTo(List.of(capturingB, capturingA, capturingButReturnsNull))); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java new file mode 100644 index 0000000..3e2b94f --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -0,0 +1,173 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoader; +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DispatchResult; +import org.dataloader.fixtures.Stopwatch; +import org.dataloader.fixtures.TestKit; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +public class DataLoaderInstrumentationTest { + + BatchLoader snoozingBatchLoader = keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(100); + return keys; + }); + + @Test + void canMonitorLoading() { + AtomicReference> dlRef = new AtomicReference<>(); + + CapturingInstrumentation instrumentation = new CapturingInstrumentation("x") { + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + DataLoaderInstrumentationContext superCtx = super.beginLoad(dataLoader, key, loadContext); + dlRef.set(dataLoader); + return superCtx; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + }; + + DataLoaderOptions options = DataLoaderOptions.newOptions() + .setInstrumentation(instrumentation) + .setMaxBatchSize(5) + .build(); + + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + List keys = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String key = "X" + i; + keys.add(key); + dl.load(key); + } + + // load a key that is cached + dl.load("X0"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(dlRef.get(), is(dl)); + assertThat(dispatch.join(), equalTo(keys)); + + // the batch loading means they start and are instrumentation dispatched before they all end up completing + assertThat(instrumentation.onlyLoads(), + equalTo(List.of( + "x_beginLoad_k:X0", "x_beginLoad_onDispatched_k:X0", + "x_beginLoad_k:X1", "x_beginLoad_onDispatched_k:X1", + "x_beginLoad_k:X2", "x_beginLoad_onDispatched_k:X2", + "x_beginLoad_k:X0", "x_beginLoad_onDispatched_k:X0", // second cached call counts + "x_beginLoad_onCompleted_k:X0", + "x_beginLoad_onCompleted_k:X0", // each load call counts + "x_beginLoad_onCompleted_k:X1", "x_beginLoad_onCompleted_k:X2"))); + + } + + + @Test + void canMonitorDispatching() { + Stopwatch stopwatch = Stopwatch.stopwatchUnStarted(); + AtomicReference> dlRef = new AtomicReference<>(); + + DataLoaderInstrumentation instrumentation = new DataLoaderInstrumentation() { + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + dlRef.set(dataLoader); + stopwatch.start(); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onCompleted(DispatchResult result, Throwable t) { + stopwatch.stop(); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + }; + + DataLoaderOptions options = DataLoaderOptions.newOptions() + .setInstrumentation(instrumentation) + .setMaxBatchSize(5) + .build(); + + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + List keys = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + String key = "X" + i; + keys.add(key); + dl.load(key); + } + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + // we must have called batch load 4 times at 100ms snooze per call + // but its in parallel via supplyAsync + assertThat(stopwatch.elapsed(), greaterThan(75L)); + assertThat(dlRef.get(), is(dl)); + assertThat(dispatch.join(), equalTo(keys)); + } + + @Test + void canMonitorBatchLoading() { + Stopwatch stopwatch = Stopwatch.stopwatchUnStarted(); + AtomicReference beRef = new AtomicReference<>(); + AtomicReference> dlRef = new AtomicReference<>(); + + DataLoaderInstrumentation instrumentation = new DataLoaderInstrumentation() { + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + dlRef.set(dataLoader); + beRef.set(environment); + + stopwatch.start(); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onCompleted(List result, Throwable t) { + stopwatch.stop(); + } + }; + } + }; + + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(instrumentation).build(); + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + dl.load("A", "kcA"); + dl.load("B", "kcB"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(stopwatch.elapsed(), greaterThan(50L)); + assertThat(dlRef.get(), is(dl)); + assertThat(beRef.get().getKeyContexts().keySet(), equalTo(Set.of("A", "B"))); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java new file mode 100644 index 0000000..e672d80 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -0,0 +1,231 @@ +package org.dataloader.instrumentation; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DataLoaderRegistry; +import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.dataloader.registries.ScheduledDataLoaderRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class DataLoaderRegistryInstrumentationTest { + DataLoader dlX; + DataLoader dlY; + DataLoader dlZ; + + CapturingInstrumentation instrA; + CapturingInstrumentation instrB; + ChainedDataLoaderInstrumentation chainedInstrA; + ChainedDataLoaderInstrumentation chainedInstrB; + + @BeforeEach + void setUp() { + dlX = TestKit.idLoader("X"); + dlY = TestKit.idLoader("Y"); + dlZ = TestKit.idLoader("Z"); + instrA = new CapturingInstrumentation("A"); + instrB = new CapturingInstrumentation("B"); + chainedInstrA = new ChainedDataLoaderInstrumentation().add(instrA); + chainedInstrB = new ChainedDataLoaderInstrumentation().add(instrB); + } + + @Test + void canInstrumentRegisteredDLsViaBuilder() { + + assertThat(dlX.getOptions().getInstrumentation(), equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + assertThat(registry.getInstrumentation(), equalTo(chainedInstrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void canInstrumentRegisteredDLsViaBuilderCombined() { + + DataLoaderRegistry registry1 = DataLoaderRegistry.newRegistry() + .register("X", dlX) + .register("Y", dlY) + .build(); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .register("Z", dlZ) + .registerAll(registry1) + .build(); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void canInstrumentViaMutativeRegistration() { + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .build(); + + registry.register("X", dlX); + registry.computeIfAbsent("Y", l -> dlY); + registry.computeIfAbsent("Z", l -> dlZ); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + } + } + + @Test + void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().transform(b-> b.setInstrumentation(instrA)))); + DataLoader newY = dlY.transform(builder -> builder.options(dlY.getOptions().transform(b-> b.setInstrumentation(instrA)))); + DataLoader newZ = dlZ.transform(builder -> builder.options(dlZ.getOptions().transform(b-> b.setInstrumentation(instrA)))); + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", newX) + .register("Y", newY) + .register("Z", newZ) + .build(); + + Map> dls = Map.of("X", newX, "Y", newY, "Z", newZ); + + assertThat(registry.getInstrumentation(), equalTo(instrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoader dataLoader = registry.getDataLoader(key); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, equalTo(instrA)); + // it's the same DL - it's not changed because it has the same instrumentation + assertThat(dls.get(key), equalTo(dataLoader)); + } + } + + @Test + void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { + DataLoaderOptions options = dlX.getOptions().transform(b -> b.setInstrumentation(instrA)); + DataLoader newX = dlX.transform(builder -> builder.options(options)); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrB) + .register("X", newX) + .build(); + + DataLoader dataLoader = registry.getDataLoader("X"); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + // it gets turned into a chained one and the registry one goes first + assertThat(instrumentations, equalTo(List.of(instrB, instrA))); + } + + @Test + void chainedInstrumentationsWillBeCombined() { + DataLoaderOptions options = dlX.getOptions().transform(b -> b.setInstrumentation(chainedInstrB)); + DataLoader newX = dlX.transform(builder -> builder.options(options)); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", newX) + .build(); + + DataLoader dataLoader = registry.getDataLoader("X"); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + // it gets turned into a chained one and the registry one goes first + assertThat(instrumentations, equalTo(List.of(instrA, instrB))); + } + + @SuppressWarnings("resource") + @Test + void canInstrumentScheduledRegistryViaBuilder() { + + assertThat(dlX.getOptions().getInstrumentation(), equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .instrumentation(chainedInstrA) + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + assertThat(registry.getInstrumentation(), equalTo(chainedInstrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void endToEndIntegrationTest(TestDataLoaderFactory factory) { + DataLoader dl = factory.idLoader(); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", dl) + .build(); + + // since the data-loader changed when registered you MUST get the data loader from the registry + // not direct to the old one + DataLoader dataLoader = registry.getDataLoader("X"); + CompletableFuture loadA = dataLoader.load("A"); + + registry.dispatchAll(); + + await().until(loadA::isDone); + assertThat(loadA.join(), equalTo("A")); + + assertThat(instrA.notLoads(), equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java b/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java new file mode 100644 index 0000000..38328eb --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java @@ -0,0 +1,49 @@ +package org.dataloader.instrumentation; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.nullValue; + +public class SimpleDataLoaderInstrumentationContextTest { + + @Test + void canRunCompletedCodeAsExpected() { + AtomicReference actual = new AtomicReference<>(); + AtomicReference actualErr = new AtomicReference<>(); + + DataLoaderInstrumentationContext ctx = DataLoaderInstrumentationHelper.whenCompleted((r, err) -> { + actualErr.set(err); + actual.set(r); + }); + + ctx.onDispatched(); // nothing happens + assertThat(actual.get(), nullValue()); + assertThat(actualErr.get(), nullValue()); + + ctx.onCompleted("X", null); + assertThat(actual.get(), Matchers.equalTo("X")); + assertThat(actualErr.get(), nullValue()); + + ctx.onCompleted(null, new RuntimeException()); + assertThat(actual.get(), nullValue()); + assertThat(actualErr.get(), Matchers.instanceOf(RuntimeException.class)); + } + + @Test + void canRunOnDispatchCodeAsExpected() { + AtomicBoolean dispatchedCalled = new AtomicBoolean(); + + DataLoaderInstrumentationContext ctx = DataLoaderInstrumentationHelper.whenDispatched(() -> dispatchedCalled.set(true)); + + ctx.onCompleted("X", null); // nothing happens + assertThat(dispatchedCalled.get(), Matchers.equalTo(false)); + + ctx.onDispatched(); + assertThat(dispatchedCalled.get(), Matchers.equalTo(true)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/performance/AtomicVsAdder.java b/src/test/java/org/dataloader/performance/AtomicVsAdder.java new file mode 100644 index 0000000..ee5e02a --- /dev/null +++ b/src/test/java/org/dataloader/performance/AtomicVsAdder.java @@ -0,0 +1,111 @@ +package org.dataloader.performance; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +public class AtomicVsAdder { + + private static final ExecutorService EXECUTOR = Executors.newCachedThreadPool(); + + public static void main(final String[] args) throws Exception { + // knobs + final var iterationsList = List.of(1 << 20L, 1 << 24L); + final var numberOfThreadsList = List.of(1, 2, 4, 8, 16); + final var strategies = List.of(new LongAdderStrategy(), new AtomicLongStrategy()); + + // test + System.out.println("testing with #cpu=" + Runtime.getRuntime().availableProcessors()); + for (int iterations : iterationsList) { + for (int numberOfThreads : numberOfThreadsList) { + for (Strategy strategy : strategies) { + performTest(iterations, numberOfThreads, strategy); + } + } + } + + EXECUTOR.shutdownNow(); + + } + + private static void performTest(final long iterations, final int numberOfThreads, Strategy strategy) throws Exception { + final List> futures = new ArrayList<>(); + System.out.println("start test with " + iterations + " iterations using " + numberOfThreads + " threads and strategy " + strategy.getClass().getSimpleName()); + final long start = System.nanoTime(); + + for (int i = 0; i < numberOfThreads; i++) { + Future submit = EXECUTOR.submit(() -> concurrentWork(strategy, iterations)); + futures.add(submit); + } + for (final Future future : futures) { + future.get(); // wait for all + } + final long end = System.nanoTime(); + System.out.println("done in " + Duration.ofNanos(end - start).toMillis() + "ms => result " + strategy.get()); + System.out.println("----"); + strategy.reset(); + } + + @SuppressWarnings("SameParameterValue") + private static void concurrentWork(final Strategy strategy, final long iterations) { + long work = iterations; + while (work-- > 0) { + strategy.increment(); + } + } + + interface Strategy { + void increment(); + + long get(); + + void reset(); + } + + static class LongAdderStrategy implements Strategy { + + private LongAdder longAdder = new LongAdder(); + + @Override + public void increment() { + longAdder.increment(); + } + + @Override + public long get() { + return longAdder.sum(); + } + + @Override + public void reset() { + longAdder = new LongAdder(); + } + } + + static class AtomicLongStrategy implements Strategy { + + private final AtomicLong atomicLong = new AtomicLong(0); + + @Override + public void increment() { + atomicLong.incrementAndGet(); + } + + @Override + public long get() { + return atomicLong.get(); + } + + @Override + public void reset() { + atomicLong.set(0); + } + } + +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/registries/DispatchPredicateTest.java b/src/test/java/org/dataloader/registries/DispatchPredicateTest.java index f241c2f..07a7416 100644 --- a/src/test/java/org/dataloader/registries/DispatchPredicateTest.java +++ b/src/test/java/org/dataloader/registries/DispatchPredicateTest.java @@ -4,12 +4,12 @@ import org.dataloader.DataLoader; import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.TestingClock; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.time.Duration; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class DispatchPredicateTest { diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java new file mode 100644 index 0000000..94f5cff --- /dev/null +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java @@ -0,0 +1,247 @@ +package org.dataloader.registries; + +import org.dataloader.BatchLoader; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderRegistry; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +import static java.util.Arrays.asList; +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.fixtures.TestKit.asSet; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class ScheduledDataLoaderRegistryPredicateTest { + final BatchLoader identityBatchLoader = CompletableFuture::completedFuture; + + static class CountingDispatchPredicate implements DispatchPredicate { + int count = 0; + int max = 0; + + public CountingDispatchPredicate(int max) { + this.max = max; + } + + @Override + public boolean test(String dataLoaderKey, DataLoader dataLoader) { + boolean shouldFire = count >= max; + count++; + return shouldFire; + } + } + + @Test + public void predicate_registration_works() { + DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlB = newDataLoader(identityBatchLoader); + DataLoader dlC = newDataLoader(identityBatchLoader); + + DispatchPredicate predicateA = new CountingDispatchPredicate(1); + DispatchPredicate predicateB = new CountingDispatchPredicate(2); + DispatchPredicate predicateC = new CountingDispatchPredicate(3); + + DispatchPredicate predicateOverAll = new CountingDispatchPredicate(10); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA, predicateA) + .register("b", dlB, predicateB) + .register("c", dlC, predicateC) + .dispatchPredicate(predicateOverAll) + .build(); + + assertThat(registry.getDataLoaders(), equalTo(asList(dlA, dlB, dlC))); + assertThat(registry.getDataLoadersMap().keySet(), equalTo(asSet("a", "b", "c"))); + assertThat(asSet(registry.getDataLoadersMap().values()), equalTo(asSet(dlA, dlB, dlC))); + assertThat(registry.getDispatchPredicate(), equalTo(predicateOverAll)); + assertThat(asSet(registry.getDataLoaderPredicates().values()), equalTo(asSet(predicateA, predicateB, predicateC))); + + // and unregister (fluently) + DataLoaderRegistry dlR = registry.unregister("c"); + assertThat(dlR, equalTo(registry)); + + assertThat(registry.getDataLoaders(), equalTo(asList(dlA, dlB))); + assertThat(registry.getDispatchPredicate(), equalTo(predicateOverAll)); + assertThat(asSet(registry.getDataLoaderPredicates().values()), equalTo(asSet(predicateA, predicateB))); + + // direct on the registry works + registry.register("c", dlC, predicateC); + assertThat(registry.getDataLoaders(), equalTo(asList(dlA, dlB, dlC))); + assertThat(registry.getDispatchPredicate(), equalTo(predicateOverAll)); + assertThat(asSet(registry.getDataLoaderPredicates().values()), equalTo(asSet(predicateA, predicateB, predicateC))); + + } + + @Test + public void predicate_firing_works() { + DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlB = newDataLoader(identityBatchLoader); + DataLoader dlC = newDataLoader(identityBatchLoader); + + DispatchPredicate predicateA = new CountingDispatchPredicate(1); + DispatchPredicate predicateB = new CountingDispatchPredicate(2); + DispatchPredicate predicateC = new CountingDispatchPredicate(3); + + DispatchPredicate predicateOnTen = new CountingDispatchPredicate(10); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA, predicateA) + .register("b", dlB, predicateB) + .register("c", dlC, predicateC) + .dispatchPredicate(predicateOnTen) + .schedule(Duration.ofHours(1000)) // make this so long its never rescheduled + .build(); + + + CompletableFuture cfA = dlA.load("A"); + CompletableFuture cfB = dlB.load("B"); + CompletableFuture cfC = dlC.load("C"); + + int count = registry.dispatchAllWithCount(); // first firing + // none should fire + assertThat(count, equalTo(0)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // second firing + // one should fire + assertThat(count, equalTo(1)); + assertThat(cfA.isDone(), equalTo(true)); + assertThat(cfA.join(), equalTo("A")); + + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // third firing + assertThat(count, equalTo(1)); + assertThat(cfA.isDone(), equalTo(true)); + assertThat(cfB.isDone(), equalTo(true)); + assertThat(cfB.join(), equalTo("B")); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // fourth firing + assertThat(count, equalTo(1)); + assertThat(cfA.isDone(), equalTo(true)); + assertThat(cfB.isDone(), equalTo(true)); + assertThat(cfC.isDone(), equalTo(true)); + assertThat(cfC.join(), equalTo("C")); + } + + @Test + public void test_the_registry_overall_predicate_firing_works() { + DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlB = newDataLoader(identityBatchLoader); + DataLoader dlC = newDataLoader(identityBatchLoader); + + DispatchPredicate predicateOnThree = new CountingDispatchPredicate(3); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA, new CountingDispatchPredicate(99)) + .register("b", dlB, new CountingDispatchPredicate(99)) + .register("c", dlC) // has none + .dispatchPredicate(predicateOnThree) + .schedule(Duration.ofHours(1000)) + .build(); + + + CompletableFuture cfA = dlA.load("A"); + CompletableFuture cfB = dlB.load("B"); + CompletableFuture cfC = dlC.load("C"); + + int count = registry.dispatchAllWithCount(); // first firing + assertThat(count, equalTo(0)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // second firing + assertThat(count, equalTo(0)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // third firing + assertThat(count, equalTo(0)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // fourth firing + assertThat(count, equalTo(1)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); // they wont ever finish until 99 calls + assertThat(cfC.isDone(), equalTo(true)); + } + + @Test + public void dispatch_immediate_firing_works() { + DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlB = newDataLoader(identityBatchLoader); + DataLoader dlC = newDataLoader(identityBatchLoader); + + DispatchPredicate predicateA = new CountingDispatchPredicate(1); + DispatchPredicate predicateB = new CountingDispatchPredicate(2); + DispatchPredicate predicateC = new CountingDispatchPredicate(3); + + DispatchPredicate predicateOverAll = new CountingDispatchPredicate(10); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA, predicateA) + .register("b", dlB, predicateB) + .register("c", dlC, predicateC) + .dispatchPredicate(predicateOverAll) + .schedule(Duration.ofHours(1000)) + .build(); + + + CompletableFuture cfA = dlA.load("A"); + CompletableFuture cfB = dlB.load("B"); + CompletableFuture cfC = dlC.load("C"); + + int count = registry.dispatchAllWithCountImmediately(); // all should fire + assertThat(count, equalTo(3)); + assertThat(cfA.isDone(), equalTo(true)); + assertThat(cfA.join(), equalTo("A")); + assertThat(cfB.isDone(), equalTo(true)); + assertThat(cfB.join(), equalTo("B")); + assertThat(cfC.isDone(), equalTo(true)); + assertThat(cfC.join(), equalTo("C")); + } + + @Test + public void test_the_registry_overall_predicate_firing_works_when_on_schedule() { + DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlB = newDataLoader(identityBatchLoader); + DataLoader dlC = newDataLoader(identityBatchLoader); + + DispatchPredicate predicateOnTwenty = new CountingDispatchPredicate(20); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA) + .register("b", dlB) + .register("c", dlC) + .dispatchPredicate(predicateOnTwenty) + .schedule(Duration.ofMillis(5)) + .build(); + + + CompletableFuture cfA = dlA.load("A"); + CompletableFuture cfB = dlB.load("B"); + CompletableFuture cfC = dlC.load("C"); + + int count = registry.dispatchAllWithCount(); // first firing + assertThat(count, equalTo(0)); + + // the calls will be rescheduled until eventually the counting predicate returns true + await().until(cfA::isDone, is(true)); + + assertThat(cfA.isDone(), equalTo(true)); + assertThat(cfB.isDone(), equalTo(true)); + assertThat(cfC.isDone(), equalTo(true)); + } +} diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java index 527f419..e89939c 100644 --- a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java @@ -1,41 +1,54 @@ package org.dataloader.registries; -import junit.framework.TestCase; +import org.awaitility.core.ConditionTimeoutException; import org.dataloader.DataLoader; -import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderRegistry; -import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static org.dataloader.fixtures.TestKit.keysAsValues; +import static org.awaitility.Awaitility.await; +import static org.awaitility.Duration.TWO_SECONDS; import static org.dataloader.fixtures.TestKit.snooze; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; -public class ScheduledDataLoaderRegistryTest extends TestCase { +public class ScheduledDataLoaderRegistryTest { DispatchPredicate alwaysDispatch = (key, dl) -> true; DispatchPredicate neverDispatch = (key, dl) -> false; - public void test_basic_setup_works_like_a_normal_dlr() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void basic_setup_works_like_a_normal_dlr(TestDataLoaderFactory factory) { - List> aCalls = new ArrayList<>(); - List> bCalls = new ArrayList<>(); + List> aCalls = new ArrayList<>(); + List> bCalls = new ArrayList<>(); - DataLoader dlA = TestKit.idLoader(aCalls); + DataLoader dlA = factory.idLoader(aCalls); dlA.load("AK1"); dlA.load("AK2"); - DataLoader dlB = TestKit.idLoader(bCalls); + DataLoader dlB = factory.idLoader(bCalls); dlB.load("BK1"); dlB.load("BK2"); @@ -57,10 +70,12 @@ public void test_basic_setup_works_like_a_normal_dlr() { assertThat(bCalls, equalTo(singletonList(asList("BK1", "BK2")))); } - public void test_predicate_always_false() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void predicate_always_false(TestDataLoaderFactory factory) { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -86,14 +101,16 @@ public void test_predicate_always_false() { assertThat(calls.size(), equalTo(0)); } - public void test_predicate_that_eventually_returns_true() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void predicate_that_eventually_returns_true(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate neverDispatch = (key, dl) -> counter.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); CompletableFuture p1 = dlA.load("K1"); CompletableFuture p2 = dlA.load("K2"); @@ -117,9 +134,11 @@ public void test_predicate_that_eventually_returns_true() { assertTrue(p2.isDone()); } - public void test_dispatchAllWithCountImmediately() { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void dispatchAllWithCountImmediately(TestDataLoaderFactory factory) { + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -134,9 +153,11 @@ public void test_dispatchAllWithCountImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_dispatchAllImmediately() { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void dispatchAllImmediately(TestDataLoaderFactory factory) { + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -150,12 +171,14 @@ public void test_dispatchAllImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_rescheduleNow() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void rescheduleNow(TestDataLoaderFactory factory) { AtomicInteger i = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> i.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -173,12 +196,14 @@ public void test_rescheduleNow() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_it_will_take_out_the_schedule_once_it_dispatches() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void it_will_take_out_the_schedule_once_it_dispatches(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> counter.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -214,14 +239,16 @@ public void test_it_will_take_out_the_schedule_once_it_dispatches() { assertThat(calls, equalTo(asList(asList("K1", "K2"), asList("K3", "K4")))); } - public void test_close_is_a_one_way_door() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void close_is_a_one_way_door(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> { counter.incrementAndGet(); return false; }; - DataLoader dlA = TestKit.idLoader(); + DataLoader dlA = factory.idLoader(); dlA.load("K1"); dlA.load("K2"); @@ -257,4 +284,91 @@ public void test_close_is_a_one_way_door() { snooze(200); assertEquals(counter.get(), countThen + 1); } -} \ No newline at end of file + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void can_tick_after_first_dispatch_for_chain_data_loaders(TestDataLoaderFactory factory) { + + // delays much bigger than the tick rate will mean multiple calls to dispatch + DataLoader dlA = factory.idLoaderDelayed(Duration.ofMillis(100)); + DataLoader dlB = factory.idLoaderDelayed(Duration.ofMillis(200)); + + CompletableFuture chainedCF = dlA.load("AK1").thenCompose(dlB::load); + + AtomicBoolean done = new AtomicBoolean(); + chainedCF.whenComplete((v, t) -> done.set(true)); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA) + .register("b", dlB) + .dispatchPredicate(alwaysDispatch) + .schedule(Duration.ofMillis(10)) + .tickerMode(true) + .build(); + + assertThat(registry.isTickerMode(), equalTo(true)); + + int count = registry.dispatchAllWithCount(); + assertThat(count, equalTo(1)); + + await().atMost(TWO_SECONDS).untilAtomic(done, is(true)); + + registry.close(); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void chain_data_loaders_will_hang_if_not_in_ticker_mode(TestDataLoaderFactory factory) { + + // delays much bigger than the tick rate will mean multiple calls to dispatch + DataLoader dlA = factory.idLoaderDelayed(Duration.ofMillis(100)); + DataLoader dlB = factory.idLoaderDelayed(Duration.ofMillis(200)); + + CompletableFuture chainedCF = dlA.load("AK1").thenCompose(dlB::load); + + AtomicBoolean done = new AtomicBoolean(); + chainedCF.whenComplete((v, t) -> done.set(true)); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .register("a", dlA) + .register("b", dlB) + .dispatchPredicate(alwaysDispatch) + .schedule(Duration.ofMillis(10)) + .tickerMode(false) + .build(); + + assertThat(registry.isTickerMode(), equalTo(false)); + + int count = registry.dispatchAllWithCount(); + assertThat(count, equalTo(1)); + + try { + await().atMost(TWO_SECONDS).untilAtomic(done, is(true)); + fail("This should not have completed but rather timed out"); + } catch (ConditionTimeoutException expected) { + } + registry.close(); + } + + @Test + public void executors_are_shutdown() { + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry().build(); + + ScheduledExecutorService executorService = registry.getScheduledExecutorService(); + assertThat(executorService.isShutdown(), equalTo(false)); + registry.close(); + assertThat(executorService.isShutdown(), equalTo(true)); + + executorService = Executors.newSingleThreadScheduledExecutor(); + registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .scheduledExecutorService(executorService).build(); + + executorService = registry.getScheduledExecutorService(); + assertThat(executorService.isShutdown(), equalTo(false)); + registry.close(); + // if they provide the executor, we don't close it down + assertThat(executorService.isShutdown(), equalTo(false)); + + + } +} diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java new file mode 100644 index 0000000..b9a7c01 --- /dev/null +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -0,0 +1,187 @@ +package org.dataloader.scheduler; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.dataloader.fixtures.TestKit.keysAsMapOfValues; +import static org.dataloader.fixtures.TestKit.keysAsMapOfValuesWithContext; +import static org.dataloader.fixtures.TestKit.keysAsValues; +import static org.dataloader.fixtures.TestKit.keysAsValuesWithContext; +import static org.dataloader.fixtures.TestKit.snooze; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class BatchLoaderSchedulerTest { + + BatchLoaderScheduler immediateScheduling = new BatchLoaderScheduler() { + + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return scheduledCall.invoke(); + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return scheduledCall.invoke(); + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + scheduledCall.invoke(); + } + }; + + private BatchLoaderScheduler delayedScheduling(int ms) { + return new BatchLoaderScheduler() { + + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(ms); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + snooze(ms); + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(ms); + scheduledCall.invoke(); + } + }; + } + + private static void commonSetupAndSimpleAsserts(DataLoader identityLoader) { + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.join(), equalTo(1)); + assertThat(future2.join(), equalTo(2)); + } + + @Test + public void can_allow_a_simple_scheduler() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); + + DataLoader identityLoader = newDataLoader(keysAsValues(), options); + + commonSetupAndSimpleAsserts(identityLoader); + } + + @Test + public void can_allow_a_simple_scheduler_with_context() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); + + DataLoader identityLoader = newDataLoader(keysAsValuesWithContext(), options); + + commonSetupAndSimpleAsserts(identityLoader); + } + + @Test + public void can_allow_a_simple_scheduler_with_mapped_batch_load() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); + + DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValues(), options); + + commonSetupAndSimpleAsserts(identityLoader); + } + + @Test + public void can_allow_a_simple_scheduler_with_mapped_batch_load_with_context() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); + + DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValuesWithContext(), options); + + commonSetupAndSimpleAsserts(identityLoader); + } + + @Test + public void can_allow_an_async_scheduler() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(delayedScheduling(50)).build(); + + DataLoader identityLoader = newDataLoader(keysAsValues(), options); + + commonSetupAndSimpleAsserts(identityLoader); + } + + + @Test + public void can_allow_a_funky_scheduler() { + AtomicBoolean releaseTheHounds = new AtomicBoolean(); + BatchLoaderScheduler funkyScheduler = new BatchLoaderScheduler() { + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + while (!releaseTheHounds.get()) { + snooze(10); + } + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return CompletableFuture.supplyAsync(() -> { + while (!releaseTheHounds.get()) { + snooze(10); + } + return scheduledCall.invoke(); + }).thenCompose(Function.identity()); + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + CompletableFuture.supplyAsync(() -> { + snooze(10); + scheduledCall.invoke(); + return null; + }); + } + }; + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(funkyScheduler).build(); + + DataLoader identityLoader = newDataLoader(keysAsValues(), options); + + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + + identityLoader.dispatch(); + + // we can spin around for a while - nothing will happen until we release the hounds + for (int i = 0; i < 5; i++) { + assertThat(future1.isDone(), equalTo(false)); + assertThat(future2.isDone(), equalTo(false)); + snooze(50); + } + + releaseTheHounds.set(true); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.join(), equalTo(1)); + assertThat(future2.join(), equalTo(2)); + } + + +} diff --git a/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java b/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java index fbfd5e2..f1cc8d8 100644 --- a/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java +++ b/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java @@ -5,13 +5,13 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.concurrent.CompletableFuture; import static java.util.Collections.singletonList; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; public class StatisticsCollectorTest { diff --git a/src/test/java/org/dataloader/stats/StatisticsTest.java b/src/test/java/org/dataloader/stats/StatisticsTest.java index b900807..6c90907 100644 --- a/src/test/java/org/dataloader/stats/StatisticsTest.java +++ b/src/test/java/org/dataloader/stats/StatisticsTest.java @@ -1,11 +1,11 @@ package org.dataloader.stats; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; public class StatisticsTest {