Skip to content

Commit 9199d0c

Browse files
srzamfirmaibin
srzamfir
authored and
maibin
committed
BAEL-3299 - Testing a Spring Batch Job (eugenp#7982)
* BAEL-3299: First version. Broken tests * BAEL-3299: Fix tests * BAEL-3299: Include gitignore for output files * BAEL-3299: Example of writer unit test * BAEL-3299: Cleaned up and included more tests * BAEL-3299: Updated to use JobParameters * BAEL-3299: Fixed broken startup and included cleanup for tests * BAEL-3299: Fine tuned version. Fixed formatting. * BAEL-3299: Cleaned up redundant stuff * BAEL-3299: Fixed formatting * BAEL-3299: Moved source code in spring-batch module * BAEL-3299: Fixed broken tests
1 parent 3e4c964 commit 9199d0c

25 files changed

+750
-32
lines changed

‎spring-batch/.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
output.csv
1+
output.csv
2+
output.json

‎spring-batch/pom.xml

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,30 @@
1515
</parent>
1616

1717
<dependencies>
18+
19+
<!-- JAXB APIs & runtime no longer provided in JDK 11 -->
20+
<!-- see http://openjdk.java.net/jeps/320 -->
21+
<dependency>
22+
<groupId>javax.xml.bind</groupId>
23+
<artifactId>jaxb-api</artifactId>
24+
<version>${jaxb.version}</version>
25+
<scope>runtime</scope>
26+
</dependency>
27+
28+
<dependency>
29+
<groupId>org.glassfish.jaxb</groupId>
30+
<artifactId>jaxb-runtime</artifactId>
31+
<version>${jaxb.version}</version>
32+
<scope>runtime</scope>
33+
</dependency>
34+
1835
<!-- SQLite database driver -->
1936
<dependency>
2037
<groupId>org.xerial</groupId>
2138
<artifactId>sqlite-jdbc</artifactId>
2239
<version>${sqlite.version}</version>
2340
</dependency>
41+
2442
<dependency>
2543
<groupId>org.springframework</groupId>
2644
<artifactId>spring-oxm</artifactId>
@@ -32,40 +50,67 @@
3250
</exclusion>
3351
</exclusions>
3452
</dependency>
53+
3554
<dependency>
3655
<groupId>org.springframework</groupId>
3756
<artifactId>spring-jdbc</artifactId>
3857
<version>${spring.version}</version>
3958
</dependency>
59+
4060
<dependency>
4161
<groupId>org.springframework.batch</groupId>
4262
<artifactId>spring-batch-core</artifactId>
4363
<version>${spring.batch.version}</version>
4464
</dependency>
65+
4566
<dependency>
4667
<groupId>org.springframework.batch</groupId>
4768
<artifactId>spring-batch-test</artifactId>
4869
<version>${spring.batch.version}</version>
4970
</dependency>
71+
5072
<dependency>
5173
<groupId>com.opencsv</groupId>
5274
<artifactId>opencsv</artifactId>
5375
<version>${opencsv.version}</version>
5476
</dependency>
77+
78+
<dependency>
79+
<groupId>org.springframework.boot</groupId>
80+
<artifactId>spring-boot-starter-batch</artifactId>
81+
<version>${spring.boot.version}</version>
82+
</dependency>
83+
84+
<dependency>
85+
<groupId>org.hsqldb</groupId>
86+
<artifactId>hsqldb</artifactId>
87+
<version>2.5.0</version>
88+
<scope>runtime</scope>
89+
</dependency>
5590

5691
<dependency>
57-
<groupId>org.awaitility</groupId>
58-
<artifactId>awaitility</artifactId>
59-
<version>${awaitility.version}</version>
60-
<scope>test</scope>
61-
</dependency>
92+
<groupId>org.awaitility</groupId>
93+
<artifactId>awaitility</artifactId>
94+
<version>${awaitility.version}</version>
95+
<scope>test</scope>
96+
</dependency>
97+
98+
<dependency>
99+
<groupId>org.springframework.boot</groupId>
100+
<artifactId>spring-boot-starter-test</artifactId>
101+
<version>${spring.boot.version}</version>
102+
<scope>test</scope>
103+
</dependency>
104+
62105
</dependencies>
63106

64107
<properties>
65-
<spring.version>5.0.3.RELEASE</spring.version>
66-
<spring.batch.version>4.0.0.RELEASE</spring.batch.version>
108+
<spring.version>5.2.0.RELEASE</spring.version>
109+
<spring.batch.version>4.2.0.RELEASE</spring.batch.version>
110+
<spring.boot.version>2.1.9.RELEASE</spring.boot.version>
67111
<sqlite.version>3.15.1</sqlite.version>
68112
<opencsv.version>4.1</opencsv.version>
113+
<jaxb.version>2.3.1</jaxb.version>
69114
<awaitility.version>3.1.1</awaitility.version>
70115
</properties>
71116

‎spring-batch/src/main/java/org/baeldung/batch/App.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.baeldung.batch;
22

3+
import org.slf4j.Logger;
4+
import org.slf4j.LoggerFactory;
35
import org.springframework.batch.core.Job;
46
import org.springframework.batch.core.JobExecution;
57
import org.springframework.batch.core.JobParameters;
@@ -8,6 +10,9 @@
810
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
911

1012
public class App {
13+
14+
private static final Logger LOGGER = LoggerFactory.getLogger(App.class);
15+
1116
public static void main(final String[] args) {
1217
// Spring Java config
1318
final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
@@ -27,19 +32,16 @@ private static void runJob(AnnotationConfigApplicationContext context, String ba
2732
final JobLauncher jobLauncher = (JobLauncher) context.getBean("jobLauncher");
2833
final Job job = (Job) context.getBean(batchJobName);
2934

30-
System.out.println("----------------------------------------");
31-
System.out.println("Starting the batch job: " + batchJobName);
35+
LOGGER.info("Starting the batch job: {}", batchJobName);
3236
try {
33-
// To enable multiple execution of a job with the same parameters
34-
JobParameters jobParameters = new JobParametersBuilder()
35-
.addString("jobID", String.valueOf(System.currentTimeMillis()))
36-
.toJobParameters();
37+
// To enable multiple execution of a job with the same parameters
38+
JobParameters jobParameters = new JobParametersBuilder().addString("jobID", String.valueOf(System.currentTimeMillis()))
39+
.toJobParameters();
3740
final JobExecution execution = jobLauncher.run(job, jobParameters);
38-
System.out.println("Job Status : " + execution.getStatus());
39-
System.out.println("Job succeeded");
41+
LOGGER.info("Job Status : {}", execution.getStatus());
4042
} catch (final Exception e) {
4143
e.printStackTrace();
42-
System.out.println("Job failed");
44+
LOGGER.error("Job failed {}", e.getMessage());
4345
}
4446
}
4547
}

‎spring-batch/src/main/java/org/baeldung/batch/SpringBatchConfig.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,14 @@ public Marshaller marshaller() {
8686
}
8787

8888
@Bean
89-
protected Step step1(@Qualifier("itemProcessor") ItemProcessor<Transaction, Transaction> processor,
90-
ItemWriter<Transaction> writer) throws ParseException {
91-
return stepBuilderFactory.get("step1").<Transaction, Transaction>chunk(10).reader(itemReader(inputCsv)).processor(processor).writer(writer).build();
89+
protected Step step1(@Qualifier("itemProcessor") ItemProcessor<Transaction, Transaction> processor, ItemWriter<Transaction> writer) throws ParseException {
90+
return stepBuilderFactory
91+
.get("step1")
92+
.<Transaction, Transaction> chunk(10)
93+
.reader(itemReader(inputCsv))
94+
.processor(processor)
95+
.writer(writer)
96+
.build();
9297
}
9398

9499
@Bean(name = "firstBatchJob")

‎spring-batch/src/main/java/org/baeldung/batch/partitioner/SpringbatchPartitionerApp.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
package org.baeldung.batch.partitioner;
22

3+
import org.slf4j.Logger;
4+
import org.slf4j.LoggerFactory;
35
import org.springframework.batch.core.Job;
46
import org.springframework.batch.core.JobExecution;
57
import org.springframework.batch.core.JobParameters;
68
import org.springframework.batch.core.launch.JobLauncher;
79
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
810

911
public class SpringbatchPartitionerApp {
12+
13+
private static final Logger LOGGER = LoggerFactory.getLogger(SpringbatchPartitionerApp.class);
14+
1015
public static void main(final String[] args) {
1116
// Spring Java config
1217
final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
@@ -15,14 +20,13 @@ public static void main(final String[] args) {
1520

1621
final JobLauncher jobLauncher = (JobLauncher) context.getBean("jobLauncher");
1722
final Job job = (Job) context.getBean("partitionerJob");
18-
System.out.println("Starting the batch job");
23+
LOGGER.info("Starting the batch job");
1924
try {
2025
final JobExecution execution = jobLauncher.run(job, new JobParameters());
21-
System.out.println("Job Status : " + execution.getStatus());
22-
System.out.println("Job succeeded");
26+
LOGGER.info("Job Status : {}", execution.getStatus());
2327
} catch (final Exception e) {
2428
e.printStackTrace();
25-
System.out.println("Job failed");
29+
LOGGER.error("Job failed {}", e.getMessage());
2630
}
2731
}
2832
}

‎spring-batch/src/main/java/org/baeldung/batchtesting/SpringBatchApplication.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package org.baeldung.batchtesting;
2+
3+
import org.springframework.batch.core.Job;
4+
import org.springframework.batch.core.JobParametersBuilder;
5+
import org.springframework.batch.core.launch.JobLauncher;
6+
import org.springframework.beans.factory.annotation.Autowired;
7+
import org.springframework.beans.factory.annotation.Qualifier;
8+
import org.springframework.beans.factory.annotation.Value;
9+
import org.springframework.boot.CommandLineRunner;
10+
import org.springframework.boot.SpringApplication;
11+
import org.springframework.boot.autoconfigure.SpringBootApplication;
12+
import org.springframework.context.annotation.PropertySource;
13+
14+
@SpringBootApplication
15+
@PropertySource("classpath:batchtesting/application.properties")
16+
public class SpringBatchApplication implements CommandLineRunner {
17+
18+
@Autowired
19+
private JobLauncher jobLauncher;
20+
21+
@Autowired
22+
@Qualifier("transformBooksRecords")
23+
private Job transformBooksRecordsJob;
24+
25+
@Value("${file.input}")
26+
private String input;
27+
28+
@Value("${file.output}")
29+
private String output;
30+
31+
public static void main(String[] args) {
32+
SpringApplication.run(SpringBatchApplication.class, args);
33+
}
34+
35+
@Override
36+
public void run(String... args) throws Exception {
37+
JobParametersBuilder paramsBuilder = new JobParametersBuilder();
38+
paramsBuilder.addString("file.input", input);
39+
paramsBuilder.addString("file.output", output);
40+
jobLauncher.run(transformBooksRecordsJob, paramsBuilder.toJobParameters());
41+
}
42+
43+
}

‎spring-batch/src/main/java/org/baeldung/batchtesting/SpringBatchConfiguration.java

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package org.baeldung.batchtesting;
2+
3+
import java.io.IOException;
4+
5+
import org.baeldung.batchtesting.model.Book;
6+
import org.baeldung.batchtesting.model.BookDetails;
7+
import org.baeldung.batchtesting.model.BookRecord;
8+
import org.baeldung.batchtesting.service.BookDetailsItemProcessor;
9+
import org.baeldung.batchtesting.service.BookItemProcessor;
10+
import org.baeldung.batchtesting.service.BookRecordFieldSetMapper;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
import org.springframework.batch.core.Job;
14+
import org.springframework.batch.core.Step;
15+
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
16+
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
17+
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
18+
import org.springframework.batch.core.configuration.annotation.StepScope;
19+
import org.springframework.batch.item.ItemReader;
20+
import org.springframework.batch.item.ItemWriter;
21+
import org.springframework.batch.item.file.FlatFileItemReader;
22+
import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
23+
import org.springframework.batch.item.file.mapping.FieldSetMapper;
24+
import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
25+
import org.springframework.batch.item.json.JsonFileItemWriter;
26+
import org.springframework.batch.item.json.builder.JsonFileItemWriterBuilder;
27+
import org.springframework.batch.item.support.ListItemWriter;
28+
import org.springframework.beans.factory.annotation.Autowired;
29+
import org.springframework.beans.factory.annotation.Value;
30+
import org.springframework.context.annotation.Bean;
31+
import org.springframework.context.annotation.Configuration;
32+
import org.springframework.core.io.FileSystemResource;
33+
34+
35+
@Configuration
36+
@EnableBatchProcessing
37+
public class SpringBatchConfiguration {
38+
39+
private static Logger LOGGER = LoggerFactory.getLogger(SpringBatchConfiguration.class);
40+
41+
private static final String[] TOKENS = { "bookname", "bookauthor", "bookformat", "isbn", "publishyear" };
42+
43+
@Autowired
44+
private JobBuilderFactory jobBuilderFactory;
45+
46+
@Autowired
47+
private StepBuilderFactory stepBuilderFactory;
48+
49+
@Bean
50+
@StepScope
51+
public FlatFileItemReader<BookRecord> csvItemReader(@Value("#{jobParameters['file.input']}") String input) {
52+
FlatFileItemReaderBuilder<BookRecord> builder = new FlatFileItemReaderBuilder<>();
53+
FieldSetMapper<BookRecord> bookRecordFieldSetMapper = new BookRecordFieldSetMapper();
54+
LOGGER.info("Configuring reader to input {}", input);
55+
// @formatter:off
56+
return builder
57+
.name("bookRecordItemReader")
58+
.resource(new FileSystemResource(input))
59+
.delimited()
60+
.names(TOKENS)
61+
.fieldSetMapper(bookRecordFieldSetMapper)
62+
.build();
63+
// @formatter:on
64+
}
65+
66+
@Bean
67+
@StepScope
68+
public JsonFileItemWriter<Book> jsonItemWriter(@Value("#{jobParameters['file.output']}") String output) throws IOException {
69+
JsonFileItemWriterBuilder<Book> builder = new JsonFileItemWriterBuilder<>();
70+
JacksonJsonObjectMarshaller<Book> marshaller = new JacksonJsonObjectMarshaller<>();
71+
LOGGER.info("Configuring writer to output {}", output);
72+
// @formatter:off
73+
return builder
74+
.name("bookItemWriter")
75+
.jsonObjectMarshaller(marshaller)
76+
.resource(new FileSystemResource(output))
77+
.build();
78+
// @formatter:on
79+
}
80+
81+
@Bean
82+
@StepScope
83+
public ListItemWriter<BookDetails> listItemWriter() {
84+
return new ListItemWriter<BookDetails>();
85+
}
86+
87+
@Bean
88+
@StepScope
89+
public BookItemProcessor bookItemProcessor() {
90+
return new BookItemProcessor();
91+
}
92+
93+
@Bean
94+
@StepScope
95+
public BookDetailsItemProcessor bookDetailsItemProcessor() {
96+
return new BookDetailsItemProcessor();
97+
}
98+
99+
@Bean
100+
public Step step1(ItemReader<BookRecord> csvItemReader, ItemWriter<Book> jsonItemWriter) throws IOException {
101+
// @formatter:off
102+
return stepBuilderFactory
103+
.get("step1")
104+
.<BookRecord, Book> chunk(3)
105+
.reader(csvItemReader)
106+
.processor(bookItemProcessor())
107+
.writer(jsonItemWriter)
108+
.build();
109+
// @formatter:on
110+
}
111+
112+
@Bean
113+
public Step step2(ItemReader<BookRecord> csvItemReader, ItemWriter<BookDetails> listItemWriter) {
114+
// @formatter:off
115+
return stepBuilderFactory
116+
.get("step2")
117+
.<BookRecord, BookDetails> chunk(3)
118+
.reader(csvItemReader)
119+
.processor(bookDetailsItemProcessor())
120+
.writer(listItemWriter)
121+
.build();
122+
// @formatter:on
123+
}
124+
125+
@Bean(name = "transformBooksRecords")
126+
public Job transformBookRecords(Step step1, Step step2) throws IOException {
127+
// @formatter:off
128+
return jobBuilderFactory
129+
.get("transformBooksRecords")
130+
.flow(step1)
131+
.next(step2)
132+
.end()
133+
.build();
134+
// @formatter:on
135+
}
136+
137+
}

0 commit comments

Comments
 (0)