Skip to content

Commit c1c7737

Browse files
johnmccleanjohnmcclean
committed
Merge pull request aol#173 from aol/s3-cleanup
Cleanup bean for s3. Will clean required folder on startup if enabled
2 parents eab5f3f + 17757ae commit c1c7737

File tree

6 files changed

+330
-11
lines changed

6 files changed

+330
-11
lines changed

‎micro-s3/src/main/java/com/aol/micro/server/s3/CleanupFileVisitor.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package com.aol.micro.server.s3;
2+
3+
import java.io.IOException;
4+
import java.nio.file.FileVisitResult;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.nio.file.SimpleFileVisitor;
8+
import java.nio.file.attribute.BasicFileAttributes;
9+
10+
public class CleanupFileVisitor extends SimpleFileVisitor<Path> {
11+
12+
private final Path tempDirectory;
13+
14+
public CleanupFileVisitor(Path directory) {
15+
this.tempDirectory = directory;
16+
}
17+
18+
@Override
19+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
20+
Files.delete(file);
21+
return FileVisitResult.CONTINUE;
22+
}
23+
24+
@Override
25+
public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
26+
if (e == null) {
27+
if(!dir.equals(tempDirectory)) {
28+
Files.delete(dir);
29+
}
30+
return FileVisitResult.CONTINUE;
31+
} else {
32+
throw e;
33+
}
34+
}
35+
}

‎micro-s3/src/main/java/com/aol/micro/server/s3/DirectoryCleaner.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package com.aol.micro.server.s3;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.nio.file.FileSystems;
6+
import java.nio.file.Files;
7+
import java.nio.file.Path;
8+
9+
import javax.annotation.PostConstruct;
10+
11+
import org.springframework.beans.factory.annotation.Autowired;
12+
import org.springframework.beans.factory.annotation.Value;
13+
import org.springframework.stereotype.Component;
14+
15+
@Component
16+
public class DirectoryCleaner {
17+
18+
private final String temporaryDirectory;
19+
20+
@Autowired
21+
public DirectoryCleaner(@Value("${s3.temp.dir:#{null}") String temporaryDirectory) {
22+
this.temporaryDirectory = temporaryDirectory;
23+
}
24+
25+
@PostConstruct
26+
public void clean() throws IOException {
27+
if (temporaryDirectory != null && new File(temporaryDirectory).exists()) {
28+
Path directory = FileSystems.getDefault().getPath(temporaryDirectory);
29+
Files.walkFileTree(directory, new CleanupFileVisitor(directory));
30+
}
31+
}
32+
33+
}

‎micro-s3/src/main/java/com/aol/micro/server/s3/S3ObjectSummaryIterator.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package com.aol.micro.server.s3;
2+
3+
import java.util.Iterator;
4+
5+
import com.amazonaws.services.s3.AmazonS3Client;
6+
import com.amazonaws.services.s3.model.ListObjectsRequest;
7+
import com.amazonaws.services.s3.model.ObjectListing;
8+
import com.amazonaws.services.s3.model.S3ObjectSummary;
9+
10+
public class S3ObjectSummaryIterator implements Iterator<S3ObjectSummary> {
11+
12+
private final AmazonS3Client client;
13+
private ListObjectsRequest req;
14+
private Iterator<S3ObjectSummary> iterator;
15+
private boolean empty = true;
16+
public S3ObjectSummaryIterator(AmazonS3Client client, ListObjectsRequest req) {
17+
this.client = client;
18+
this.req = req;
19+
updateIterator();
20+
}
21+
22+
@Override
23+
public boolean hasNext() {
24+
if(iterator.hasNext()) {
25+
return true;
26+
} else if(!empty){
27+
updateIterator();
28+
return iterator.hasNext();
29+
} else {
30+
return false;
31+
}
32+
}
33+
34+
private void updateIterator() {
35+
if(iterator == null || !iterator.hasNext()) {
36+
ObjectListing listing = client.listObjects(req);
37+
req = req.withMarker(listing.getNextMarker());
38+
empty = !listing.isTruncated();
39+
iterator = listing.getObjectSummaries().iterator();
40+
}
41+
}
42+
43+
@Override
44+
public S3ObjectSummary next() {
45+
updateIterator();
46+
return iterator.next();
47+
}
48+
49+
}

‎micro-s3/src/main/java/com/aol/micro/server/s3/S3Utils.java

Lines changed: 85 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,56 @@
11
package com.aol.micro.server.s3;
22

3+
import java.io.ByteArrayInputStream;
4+
import java.io.File;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.nio.file.FileSystems;
8+
import java.nio.file.Files;
39
import java.util.ArrayList;
410
import java.util.List;
511
import java.util.function.Function;
6-
import java.util.stream.Stream;
12+
import java.util.function.Supplier;
713

14+
import org.apache.commons.io.FileUtils;
815
import org.springframework.beans.factory.annotation.Autowired;
16+
import org.springframework.beans.factory.annotation.Value;
917
import org.springframework.stereotype.Component;
1018

19+
import com.amazonaws.AmazonClientException;
20+
import com.amazonaws.AmazonServiceException;
1121
import com.amazonaws.services.s3.AmazonS3Client;
1222
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
1323
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
1424
import com.amazonaws.services.s3.model.ListObjectsRequest;
1525
import com.amazonaws.services.s3.model.ObjectListing;
1626
import com.amazonaws.services.s3.model.S3ObjectSummary;
27+
import com.amazonaws.services.s3.transfer.Download;
28+
import com.amazonaws.services.s3.transfer.TransferManager;
1729
import com.aol.cyclops.control.ReactiveSeq;
30+
import com.aol.cyclops.util.ExceptionSoftener;
1831

1932
@Component
2033
public class S3Utils {
2134

2235
private final AmazonS3Client client;
36+
private final TransferManager transferManager;
37+
private final String tmpDirectory;
2338

2439
@Autowired
25-
public S3Utils(AmazonS3Client client) {
40+
public S3Utils(AmazonS3Client client, TransferManager transferManager,
41+
@Value("${s3.tmp.dir:#{null}}") String tmpDirectory) {
2642
this.client = client;
43+
this.transferManager = transferManager;
44+
this.tmpDirectory = tmpDirectory;
2745
}
2846

47+
48+
/**
49+
* Method returns list of all <b>S3ObjectSummary</b> objects, subject to <i>req</i> parameters.
50+
* Multiple S3 calls will be performed if there are more than 1000 elements there
51+
* @param req - ListObjectRequest to be used.
52+
* @return List of S3ObjectSummary from bucket,
53+
*/
2954
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
3055
List<S3ObjectSummary> result = new ArrayList<>();
3156
String marker = null;
@@ -36,16 +61,27 @@ public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
3661
marker = listing.getNextMarker();
3762
result.addAll(listing.getObjectSummaries());
3863
} while (listing.isTruncated());
64+
3965
return result;
4066
}
4167

42-
/*
43-
* TODO implement smarter mechanism to reduce number of queries
68+
/**
69+
* Method return stream of S3ObjectSummary objects, subject to <i>req</i> parameters
70+
* Method will perform one query for every 1000 elements (current s3 limitation).
71+
* It is lazy, so there would be no unnecesarry calls
72+
* @param req - ListObjectRequest to be used.
73+
* @param processor - Function that convert S3ObjectSummary to any object
74+
* @return ReactiveSeq of converted S3Object summary elements.
4475
*/
45-
public <T> Stream<T> getSummariesStream(ListObjectsRequest req, Function<S3ObjectSummary, T> processor) {
46-
return getAllSummaries(req).stream().map(processor);
76+
public <T> ReactiveSeq<T> getSummariesStream(ListObjectsRequest req, Function<S3ObjectSummary, T> processor) {
77+
return ReactiveSeq.fromIterator(new S3ObjectSummaryIterator(client, req)).map(processor);
4778
}
4879

80+
/**
81+
* Method delete all <i>objects</i> from <i>bucketName</i> in groups by 1000 elements
82+
* @param bucketName
83+
* @param objects
84+
*/
4985
public void delete(String bucketName, List<KeyVersion> objects) {
5086
ReactiveSeq.fromList(objects).grouped(1000).forEach(l -> {
5187
DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);
@@ -54,4 +90,47 @@ public void delete(String bucketName, List<KeyVersion> objects) {
5490
});
5591
}
5692

93+
/**
94+
* Method returns InputStream from S3Object. Multi-part download is used to get file.
95+
* s3.tmp.dir property used to store temporary files. You can specify temporary file name by
96+
* using tempFileSupplier object.
97+
* @param bucketName
98+
* @param key -
99+
* @param tempFileSupplier - Supplier providing temporary filenames
100+
* @return InputStream of
101+
* @throws AmazonServiceException
102+
* @throws AmazonClientException
103+
* @throws InterruptedException
104+
* @throws IOException
105+
*/
106+
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
107+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
108+
File file = tempFileSupplier.get();
109+
try {
110+
Download download = transferManager.download(bucketName, key, file);
111+
download.waitForCompletion();
112+
return new ByteArrayInputStream(FileUtils.readFileToByteArray(file));
113+
} finally {
114+
file.delete();
115+
}
116+
}
117+
118+
/**
119+
* Method returns InputStream from S3Object. Multi-part download is used to get file.
120+
* s3.tmp.dir property used to store temporary files.
121+
* @param bucketName
122+
* @param key
123+
* @return
124+
* @throws AmazonServiceException
125+
* @throws AmazonClientException
126+
* @throws InterruptedException
127+
* @throws IOException
128+
*/
129+
public InputStream getInputStream(String bucketName, String key)
130+
throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
131+
Supplier<File> tempFileSupplier = ExceptionSoftener.softenSupplier(() -> Files
132+
.createTempFile(FileSystems.getDefault().getPath(tmpDirectory), "micro-s3", "file").toFile());
133+
return getInputStream(bucketName, key, tempFileSupplier);
134+
}
135+
57136
}

‎micro-s3/src/test/java/com/aol/micro/server/s3/DirectoryCleanerTest.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.aol.micro.server.s3;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
7+
import org.junit.Assert;
8+
import org.junit.Test;
9+
10+
public class DirectoryCleanerTest {
11+
12+
@Test
13+
public void clean() throws IOException {
14+
15+
16+
Path dir = Files.createTempDirectory("test");
17+
DirectoryCleaner cleaner = new DirectoryCleaner(dir.toString());
18+
Path file = Files.createTempFile(dir, "a", "b");
19+
Assert.assertTrue(Files.exists(file));
20+
cleaner.clean();
21+
Assert.assertFalse(Files.exists(file));
22+
23+
}
24+
}

0 commit comments

Comments
 (0)