Skip to content

Commit 7120b58

Browse files
johnmccleanjohnmcclean
authored
Merge pull request aol#299 from aol/s3-aes256-encryption-support
Fixing default property, adding latency test
2 parents 1e5abda + 3ceec83 commit 7120b58

File tree

9 files changed

+403
-102
lines changed

9 files changed

+403
-102
lines changed

‎micro-s3/build.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ dependencies {
55
compile 'com.amazonaws:aws-java-sdk:' + s3Version
66
compile project(':micro-core')
77
compile project(':micro-manifest-comparator')
8+
testCompile group: 'org.springframework', name: 'spring-test', version: '4.0.5.RELEASE'
9+
testCompile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0'
10+
testCompile group: 'org.coursera', name: 'dropwizard-metrics-datadog', version: '1.1.6'
11+
testCompile group: 'com.github.cbismuth', name: 'junit-repeat-rule', version: '1.1.0'
812

913
compile 'commons-io:commons-io:'+commonsIOVersion
1014
testCompile project(':micro-grizzly-with-jersey')

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/ReadUtils.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package com.aol.micro.server.s3.data;
2+
3+
import java.io.ByteArrayInputStream;
4+
import java.io.File;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.nio.file.FileSystems;
8+
import java.nio.file.Files;
9+
import java.util.function.Supplier;
10+
11+
import org.apache.commons.io.FileUtils;
12+
13+
import com.amazonaws.AmazonClientException;
14+
import com.amazonaws.AmazonServiceException;
15+
import com.amazonaws.services.s3.transfer.Download;
16+
import com.amazonaws.services.s3.transfer.TransferManager;
17+
import com.aol.cyclops.util.ExceptionSoftener;
18+
19+
import lombok.AllArgsConstructor;
20+
21+
@AllArgsConstructor
22+
public class ReadUtils {
23+
private final TransferManager transferManager;
24+
private final String tmpDirectory;
25+
26+
/**
27+
* Method returns InputStream from S3Object. Multi-part download is used to
28+
* get file. s3.tmp.dir property used to store temporary files. You can
29+
* specify temporary file name by using tempFileSupplier object.
30+
*
31+
* @param bucketName
32+
* @param key
33+
* -
34+
* @param tempFileSupplier
35+
* - Supplier providing temporary filenames
36+
* @return InputStream of
37+
* @throws AmazonServiceException
38+
* @throws AmazonClientException
39+
* @throws InterruptedException
40+
* @throws IOException
41+
*/
42+
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
43+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
44+
File file = tempFileSupplier.get();
45+
try {
46+
Download download = transferManager.download(bucketName, key, file);
47+
download.waitForCompletion();
48+
return new ByteArrayInputStream(
49+
FileUtils.readFileToByteArray(file));
50+
} finally {
51+
file.delete();
52+
}
53+
}
54+
55+
/**
56+
* Method returns InputStream from S3Object. Multi-part download is used to
57+
* get file. s3.tmp.dir property used to store temporary files.
58+
*
59+
* @param bucketName
60+
* @param key
61+
* @return
62+
* @throws AmazonServiceException
63+
* @throws AmazonClientException
64+
* @throws InterruptedException
65+
* @throws IOException
66+
*/
67+
public InputStream getInputStream(String bucketName, String key)
68+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
69+
Supplier<File> tempFileSupplier = ExceptionSoftener.softenSupplier(() -> Files.createTempFile(FileSystems.getDefault()
70+
.getPath(tmpDirectory),
71+
"micro-s3",
72+
"file")
73+
.toFile());
74+
return getInputStream(bucketName, key, tempFileSupplier);
75+
}
76+
}

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/S3ObjectWriter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ public class S3ObjectWriter {
2828
private final TransferManager manager;
2929
private final String bucket;
3030
private final File dir;
31-
private final Random r = new Random();
3231
private final boolean aes256Encryption;
32+
private final Random r = new Random();
3333

3434
/**
3535
*

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/S3Reader.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
@AllArgsConstructor
1515
public class S3Reader {
1616

17-
private final S3Utils utils;
17+
private final ReadUtils readUtils;
1818
private final AmazonS3Client client;
1919
private final String bucket;
2020

@@ -34,7 +34,7 @@ public Date getLastModified(String key) {
3434
public Try<String, Throwable> getAsString(String key) {
3535
return Try.withCatch(() -> ReactiveSeq.fromStream(new BufferedReader(
3636
new InputStreamReader(
37-
utils.getInputStream(bucket,
37+
readUtils.getInputStream(bucket,
3838
key))).lines())
3939
.join("\n"));
4040

@@ -43,7 +43,7 @@ public Try<String, Throwable> getAsString(String key) {
4343
public <T> Try<T, Throwable> getAsObject(String key) {
4444
return Try.withCatch(() -> {
4545
ObjectInputStream ois = new ObjectInputStream(
46-
utils.getInputStream(bucket, key));
46+
readUtils.getInputStream(bucket, key));
4747
return (T) ois.readObject();
4848
});
4949

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/S3Utils.java

Lines changed: 4 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,27 @@
11
package com.aol.micro.server.s3.data;
22

3-
import java.io.ByteArrayInputStream;
43
import java.io.File;
54
import java.io.IOException;
65
import java.io.InputStream;
7-
import java.nio.file.FileSystems;
8-
import java.nio.file.Files;
96
import java.util.ArrayList;
107
import java.util.Iterator;
118
import java.util.List;
12-
import java.util.Random;
139
import java.util.concurrent.ExecutorService;
1410
import java.util.function.Function;
15-
import java.util.function.Supplier;
1611

17-
import org.apache.commons.io.FileUtils;
1812
import org.springframework.beans.factory.annotation.Autowired;
1913
import org.springframework.beans.factory.annotation.Qualifier;
2014
import org.springframework.beans.factory.annotation.Value;
2115
import org.springframework.stereotype.Component;
2216

23-
import com.amazonaws.AmazonClientException;
24-
import com.amazonaws.AmazonServiceException;
2517
import com.amazonaws.services.s3.AmazonS3Client;
2618
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
2719
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
2820
import com.amazonaws.services.s3.model.ListObjectsRequest;
2921
import com.amazonaws.services.s3.model.ObjectListing;
3022
import com.amazonaws.services.s3.model.S3ObjectSummary;
31-
import com.amazonaws.services.s3.transfer.Download;
3223
import com.amazonaws.services.s3.transfer.TransferManager;
3324
import com.aol.cyclops.control.ReactiveSeq;
34-
import com.aol.cyclops.util.ExceptionSoftener;
3525

3626
@Component
3727
public class S3Utils {
@@ -43,23 +33,25 @@ public class S3Utils {
4333
private final String tmpDirectory;
4434
private final ExecutorService uploaderService;
4535
private final boolean aes256Encryption;
36+
private final ReadUtils readUtils;
4637

4738
@Autowired
4839
public S3Utils(AmazonS3Client client, TransferManager transferManager,
4940
@Value("${s3.tmp.dir:#{systemProperties['java.io.tmpdir']}}") String tmpDirectory,
50-
@Value("${s3.aes256.enabled:false}}") boolean aes256Encryption,
41+
@Value("${s3.aes256.enabled:false}") boolean aes256Encryption,
5142
@Qualifier("s3UploadExecutorService") ExecutorService uploaderService) {
5243
this.client = client;
5344
this.transferManager = transferManager;
5445
this.tmpDirectory = tmpDirectory;
5546
this.uploaderService = uploaderService;
5647
this.aes256Encryption = aes256Encryption;
48+
this.readUtils = new ReadUtils(transferManager, tmpDirectory);
5749
}
5850

5951
public S3Reader reader(String bucket) {
6052

6153
return new S3Reader(
62-
this, client, bucket);
54+
readUtils, client, bucket);
6355
}
6456

6557
public S3ObjectWriter writer(String bucket) {
@@ -136,57 +128,6 @@ public void delete(String bucketName, List<KeyVersion> objects) {
136128
});
137129
}
138130

139-
/**
140-
* Method returns InputStream from S3Object. Multi-part download is used to
141-
* get file. s3.tmp.dir property used to store temporary files. You can
142-
* specify temporary file name by using tempFileSupplier object.
143-
*
144-
* @param bucketName
145-
* @param key
146-
* -
147-
* @param tempFileSupplier
148-
* - Supplier providing temporary filenames
149-
* @return InputStream of
150-
* @throws AmazonServiceException
151-
* @throws AmazonClientException
152-
* @throws InterruptedException
153-
* @throws IOException
154-
*/
155-
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
156-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
157-
File file = tempFileSupplier.get();
158-
try {
159-
Download download = transferManager.download(bucketName, key, file);
160-
download.waitForCompletion();
161-
return new ByteArrayInputStream(
162-
FileUtils.readFileToByteArray(file));
163-
} finally {
164-
file.delete();
165-
}
166-
}
167-
168-
/**
169-
* Method returns InputStream from S3Object. Multi-part download is used to
170-
* get file. s3.tmp.dir property used to store temporary files.
171-
*
172-
* @param bucketName
173-
* @param key
174-
* @return
175-
* @throws AmazonServiceException
176-
* @throws AmazonClientException
177-
* @throws InterruptedException
178-
* @throws IOException
179-
*/
180-
public InputStream getInputStream(String bucketName, String key)
181-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
182-
Supplier<File> tempFileSupplier = ExceptionSoftener.softenSupplier(() -> Files.createTempFile(FileSystems.getDefault()
183-
.getPath(tmpDirectory),
184-
"micro-s3",
185-
"file")
186-
.toFile());
187-
return getInputStream(bucketName, key, tempFileSupplier);
188-
}
189-
190131
/**
191132
* Provide empty InputStream.
192133
* <p>

‎micro-s3/src/test/java/com/aol/micro/server/s3/ReadUtilsTest.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.aol.micro.server.s3;
2+
3+
import static org.mockito.Matchers.any;
4+
import static org.mockito.Matchers.anyString;
5+
import static org.mockito.Mockito.mock;
6+
import static org.mockito.Mockito.verify;
7+
import static org.mockito.Mockito.when;
8+
9+
import java.io.File;
10+
import java.io.IOException;
11+
import java.nio.file.Files;
12+
13+
import org.junit.Assert;
14+
import org.junit.Test;
15+
16+
import com.amazonaws.AmazonClientException;
17+
import com.amazonaws.AmazonServiceException;
18+
import com.amazonaws.services.s3.transfer.Download;
19+
import com.amazonaws.services.s3.transfer.TransferManager;
20+
import com.aol.micro.server.s3.data.ReadUtils;
21+
22+
public class ReadUtilsTest {
23+
24+
@Test
25+
public void getInputStreamSupplier()
26+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
27+
TransferManager transferManager = mock(TransferManager.class);
28+
Download download = mock(Download.class);
29+
30+
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
31+
32+
File file = Files.createTempFile("micro-s3", "test")
33+
.toFile();
34+
Assert.assertTrue(file.exists());
35+
ReadUtils utils = new ReadUtils(
36+
transferManager, "test");
37+
38+
utils.getInputStream("", "", () -> file);
39+
40+
Assert.assertFalse(file.exists());
41+
}
42+
43+
@Test
44+
public void getInputStreamDefaultSupplier()
45+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
46+
TransferManager transferManager = mock(TransferManager.class);
47+
Download download = mock(Download.class);
48+
49+
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
50+
51+
ReadUtils utils = new ReadUtils(
52+
transferManager, System.getProperty("java.io.tmpdir"));
53+
utils.getInputStream("", "");
54+
verify(download).waitForCompletion();
55+
}
56+
}

‎micro-s3/src/test/java/com/aol/micro/server/s3/S3UtilsTest.java

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -143,41 +143,6 @@ public void deleteObjects() {
143143
verify(client, times(2)).deleteObjects(any());
144144
}
145145

146-
@Test
147-
public void getInputStreamSupplier()
148-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
149-
AmazonS3Client client = mock(AmazonS3Client.class);
150-
TransferManager transferManager = mock(TransferManager.class);
151-
Download download = mock(Download.class);
152-
153-
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
154-
155-
File file = Files.createTempFile("micro-s3", "test")
156-
.toFile();
157-
Assert.assertTrue(file.exists());
158-
S3Utils utils = new S3Utils(
159-
client, transferManager, "test", false, null);
160-
161-
utils.getInputStream("", "", () -> file);
162-
163-
Assert.assertFalse(file.exists());
164-
}
165-
166-
@Test
167-
public void getInputStreamDefaultSupplier()
168-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
169-
AmazonS3Client client = mock(AmazonS3Client.class);
170-
TransferManager transferManager = mock(TransferManager.class);
171-
Download download = mock(Download.class);
172-
173-
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
174-
175-
S3Utils utils = new S3Utils(
176-
client, transferManager, System.getProperty("java.io.tmpdir"), false, null);
177-
utils.getInputStream("", "");
178-
verify(download).waitForCompletion();
179-
}
180-
181146
@Test
182147
public void emptyInputStream() throws IOException {
183148
assertEquals(0, S3Utils.emptyInputStream()

0 commit comments

Comments
 (0)