Skip to content

Commit 9d5637e

Browse files
author
Guzman, David
committed
Download test, fix upload test. Removing lana related variable
1 parent 6186082 commit 9d5637e

File tree

7 files changed

+290
-127
lines changed

7 files changed

+290
-127
lines changed

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/ReadUtils.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package com.aol.micro.server.s3.data;
2+
3+
import java.io.ByteArrayInputStream;
4+
import java.io.File;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.nio.file.FileSystems;
8+
import java.nio.file.Files;
9+
import java.util.function.Supplier;
10+
11+
import org.apache.commons.io.FileUtils;
12+
13+
import com.amazonaws.AmazonClientException;
14+
import com.amazonaws.AmazonServiceException;
15+
import com.amazonaws.services.s3.transfer.Download;
16+
import com.amazonaws.services.s3.transfer.TransferManager;
17+
import com.aol.cyclops.util.ExceptionSoftener;
18+
19+
import lombok.AllArgsConstructor;
20+
21+
@AllArgsConstructor
22+
public class ReadUtils {
23+
private TransferManager transferManager;
24+
private String tmpDirectory;
25+
26+
/**
27+
* Method returns InputStream from S3Object. Multi-part download is used to
28+
* get file. s3.tmp.dir property used to store temporary files. You can
29+
* specify temporary file name by using tempFileSupplier object.
30+
*
31+
* @param bucketName
32+
* @param key
33+
* -
34+
* @param tempFileSupplier
35+
* - Supplier providing temporary filenames
36+
* @return InputStream of
37+
* @throws AmazonServiceException
38+
* @throws AmazonClientException
39+
* @throws InterruptedException
40+
* @throws IOException
41+
*/
42+
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
43+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
44+
File file = tempFileSupplier.get();
45+
try {
46+
Download download = transferManager.download(bucketName, key, file);
47+
download.waitForCompletion();
48+
return new ByteArrayInputStream(
49+
FileUtils.readFileToByteArray(file));
50+
} finally {
51+
file.delete();
52+
}
53+
}
54+
55+
/**
56+
* Method returns InputStream from S3Object. Multi-part download is used to
57+
* get file. s3.tmp.dir property used to store temporary files.
58+
*
59+
* @param bucketName
60+
* @param key
61+
* @return
62+
* @throws AmazonServiceException
63+
* @throws AmazonClientException
64+
* @throws InterruptedException
65+
* @throws IOException
66+
*/
67+
public InputStream getInputStream(String bucketName, String key)
68+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
69+
Supplier<File> tempFileSupplier = ExceptionSoftener.softenSupplier(() -> Files.createTempFile(FileSystems.getDefault()
70+
.getPath(tmpDirectory),
71+
"micro-s3",
72+
"file")
73+
.toFile());
74+
return getInputStream(bucketName, key, tempFileSupplier);
75+
}
76+
}

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/S3Reader.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
@AllArgsConstructor
1515
public class S3Reader {
1616

17-
private final S3Utils utils;
17+
private final ReadUtils readUtils;
1818
private final AmazonS3Client client;
1919
private final String bucket;
2020

@@ -34,7 +34,7 @@ public Date getLastModified(String key) {
3434
public Try<String, Throwable> getAsString(String key) {
3535
return Try.withCatch(() -> ReactiveSeq.fromStream(new BufferedReader(
3636
new InputStreamReader(
37-
utils.getInputStream(bucket,
37+
readUtils.getInputStream(bucket,
3838
key))).lines())
3939
.join("\n"));
4040

@@ -43,7 +43,7 @@ public Try<String, Throwable> getAsString(String key) {
4343
public <T> Try<T, Throwable> getAsObject(String key) {
4444
return Try.withCatch(() -> {
4545
ObjectInputStream ois = new ObjectInputStream(
46-
utils.getInputStream(bucket, key));
46+
readUtils.getInputStream(bucket, key));
4747
return (T) ois.readObject();
4848
});
4949

‎micro-s3/src/main/java/com/aol/micro/server/s3/data/S3Utils.java

Lines changed: 3 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,27 @@
11
package com.aol.micro.server.s3.data;
22

3-
import java.io.ByteArrayInputStream;
43
import java.io.File;
54
import java.io.IOException;
65
import java.io.InputStream;
7-
import java.nio.file.FileSystems;
8-
import java.nio.file.Files;
96
import java.util.ArrayList;
107
import java.util.Iterator;
118
import java.util.List;
129
import java.util.concurrent.ExecutorService;
1310
import java.util.function.Function;
14-
import java.util.function.Supplier;
1511

16-
import org.apache.commons.io.FileUtils;
1712
import org.springframework.beans.factory.annotation.Autowired;
1813
import org.springframework.beans.factory.annotation.Qualifier;
1914
import org.springframework.beans.factory.annotation.Value;
2015
import org.springframework.stereotype.Component;
2116

22-
import com.amazonaws.AmazonClientException;
23-
import com.amazonaws.AmazonServiceException;
2417
import com.amazonaws.services.s3.AmazonS3Client;
2518
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
2619
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
2720
import com.amazonaws.services.s3.model.ListObjectsRequest;
2821
import com.amazonaws.services.s3.model.ObjectListing;
2922
import com.amazonaws.services.s3.model.S3ObjectSummary;
30-
import com.amazonaws.services.s3.transfer.Download;
3123
import com.amazonaws.services.s3.transfer.TransferManager;
3224
import com.aol.cyclops.control.ReactiveSeq;
33-
import com.aol.cyclops.util.ExceptionSoftener;
3425

3526
@Component
3627
public class S3Utils {
@@ -42,6 +33,7 @@ public class S3Utils {
4233
private final String tmpDirectory;
4334
private final ExecutorService uploaderService;
4435
private final boolean aes256Encryption;
36+
private final ReadUtils readUtils;
4537

4638
@Autowired
4739
public S3Utils(AmazonS3Client client, TransferManager transferManager,
@@ -53,12 +45,13 @@ public S3Utils(AmazonS3Client client, TransferManager transferManager,
5345
this.tmpDirectory = tmpDirectory;
5446
this.uploaderService = uploaderService;
5547
this.aes256Encryption = aes256Encryption;
48+
this.readUtils = new ReadUtils(transferManager, tmpDirectory);
5649
}
5750

5851
public S3Reader reader(String bucket) {
5952

6053
return new S3Reader(
61-
this, client, bucket);
54+
readUtils, client, bucket);
6255
}
6356

6457
public S3ObjectWriter writer(String bucket) {
@@ -135,57 +128,6 @@ public void delete(String bucketName, List<KeyVersion> objects) {
135128
});
136129
}
137130

138-
/**
139-
* Method returns InputStream from S3Object. Multi-part download is used to
140-
* get file. s3.tmp.dir property used to store temporary files. You can
141-
* specify temporary file name by using tempFileSupplier object.
142-
*
143-
* @param bucketName
144-
* @param key
145-
* -
146-
* @param tempFileSupplier
147-
* - Supplier providing temporary filenames
148-
* @return InputStream of
149-
* @throws AmazonServiceException
150-
* @throws AmazonClientException
151-
* @throws InterruptedException
152-
* @throws IOException
153-
*/
154-
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
155-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
156-
File file = tempFileSupplier.get();
157-
try {
158-
Download download = transferManager.download(bucketName, key, file);
159-
download.waitForCompletion();
160-
return new ByteArrayInputStream(
161-
FileUtils.readFileToByteArray(file));
162-
} finally {
163-
file.delete();
164-
}
165-
}
166-
167-
/**
168-
* Method returns InputStream from S3Object. Multi-part download is used to
169-
* get file. s3.tmp.dir property used to store temporary files.
170-
*
171-
* @param bucketName
172-
* @param key
173-
* @return
174-
* @throws AmazonServiceException
175-
* @throws AmazonClientException
176-
* @throws InterruptedException
177-
* @throws IOException
178-
*/
179-
public InputStream getInputStream(String bucketName, String key)
180-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
181-
Supplier<File> tempFileSupplier = ExceptionSoftener.softenSupplier(() -> Files.createTempFile(FileSystems.getDefault()
182-
.getPath(tmpDirectory),
183-
"micro-s3",
184-
"file")
185-
.toFile());
186-
return getInputStream(bucketName, key, tempFileSupplier);
187-
}
188-
189131
/**
190132
* Provide empty InputStream.
191133
* <p>

‎micro-s3/src/test/java/com/aol/micro/server/s3/ReadUtilsTest.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.aol.micro.server.s3;
2+
3+
import static org.mockito.Matchers.any;
4+
import static org.mockito.Matchers.anyString;
5+
import static org.mockito.Mockito.mock;
6+
import static org.mockito.Mockito.verify;
7+
import static org.mockito.Mockito.when;
8+
9+
import java.io.File;
10+
import java.io.IOException;
11+
import java.nio.file.Files;
12+
13+
import org.junit.Assert;
14+
import org.junit.Test;
15+
16+
import com.amazonaws.AmazonClientException;
17+
import com.amazonaws.AmazonServiceException;
18+
import com.amazonaws.services.s3.transfer.Download;
19+
import com.amazonaws.services.s3.transfer.TransferManager;
20+
import com.aol.micro.server.s3.data.ReadUtils;
21+
22+
public class ReadUtilsTest {
23+
24+
@Test
25+
public void getInputStreamSupplier()
26+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
27+
TransferManager transferManager = mock(TransferManager.class);
28+
Download download = mock(Download.class);
29+
30+
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
31+
32+
File file = Files.createTempFile("micro-s3", "test")
33+
.toFile();
34+
Assert.assertTrue(file.exists());
35+
ReadUtils utils = new ReadUtils(
36+
transferManager, "test");
37+
38+
utils.getInputStream("", "", () -> file);
39+
40+
Assert.assertFalse(file.exists());
41+
}
42+
43+
@Test
44+
public void getInputStreamDefaultSupplier()
45+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
46+
TransferManager transferManager = mock(TransferManager.class);
47+
Download download = mock(Download.class);
48+
49+
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
50+
51+
ReadUtils utils = new ReadUtils(
52+
transferManager, System.getProperty("java.io.tmpdir"));
53+
utils.getInputStream("", "");
54+
verify(download).waitForCompletion();
55+
}
56+
}

‎micro-s3/src/test/java/com/aol/micro/server/s3/S3UtilsTest.java

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -143,41 +143,6 @@ public void deleteObjects() {
143143
verify(client, times(2)).deleteObjects(any());
144144
}
145145

146-
@Test
147-
public void getInputStreamSupplier()
148-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
149-
AmazonS3Client client = mock(AmazonS3Client.class);
150-
TransferManager transferManager = mock(TransferManager.class);
151-
Download download = mock(Download.class);
152-
153-
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
154-
155-
File file = Files.createTempFile("micro-s3", "test")
156-
.toFile();
157-
Assert.assertTrue(file.exists());
158-
S3Utils utils = new S3Utils(
159-
client, transferManager, "test", false, null);
160-
161-
utils.getInputStream("", "", () -> file);
162-
163-
Assert.assertFalse(file.exists());
164-
}
165-
166-
@Test
167-
public void getInputStreamDefaultSupplier()
168-
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
169-
AmazonS3Client client = mock(AmazonS3Client.class);
170-
TransferManager transferManager = mock(TransferManager.class);
171-
Download download = mock(Download.class);
172-
173-
when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);
174-
175-
S3Utils utils = new S3Utils(
176-
client, transferManager, System.getProperty("java.io.tmpdir"), false, null);
177-
utils.getInputStream("", "");
178-
verify(download).waitForCompletion();
179-
}
180-
181146
@Test
182147
public void emptyInputStream() throws IOException {
183148
assertEquals(0, S3Utils.emptyInputStream()

0 commit comments

Comments
 (0)