Method for deleting objects in batches

Nikita Sapozhnikov · Nikita Sapozhnikov · commit 67e01c47900b · 2016-03-22T18:10:12.000Z
diff --git a/micro-s3/build.gradle b/micro-s3/build.gradle
@@ -1,6 +1,7 @@
 description = 'micro-s3'
 
 dependencies {
+	compile ("com.aol.simplereact:cyclops-react:$cyclopsReactVersion")
 	compile 'com.amazonaws:aws-java-sdk:' + s3Version
 	compile project(':micro-core')
 	
diff --git a/micro-s3/src/main/java/com/aol/micro/server/s3/S3Utils.java b/micro-s3/src/main/java/com/aol/micro/server/s3/S3Utils.java
@@ -9,9 +9,12 @@
 import org.springframework.stereotype.Component;
 
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.aol.cyclops.control.ReactiveSeq;
 
 @Component
 public class S3Utils {
@@ -35,11 +38,20 @@ public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
 		} while (listing.isTruncated());
 		return result;
 	}
+
 	/*
 	 * TODO implement smarter mechanism to reduce number of queries
 	 */
 	public <T> Stream<T> getSummariesStream(ListObjectsRequest req, Function<S3ObjectSummary, T> processor) {
 		return getAllSummaries(req).stream().map(processor);
 	}
 
+	public void delete(String bucketName, List<KeyVersion> objects) {
+		ReactiveSeq.fromList(objects).grouped(1000).forEach(l -> {
+			DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);
+			req.setKeys(l.toList());
+			client.deleteObjects(req);
+		});
+	}
+
 }
diff --git a/micro-s3/src/test/java/com/aol/micro/server/s3/S3UtilsTest.java b/micro-s3/src/test/java/com/aol/micro/server/s3/S3UtilsTest.java
@@ -6,9 +6,13 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.junit.Test;
 
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 
@@ -54,4 +58,18 @@ public void getSummariesStream() {
 		utils.getSummariesStream(new ListObjectsRequest(), s -> s.getKey());
 		verify(objectListing, times(2)).getObjectSummaries();
 	}
+
+	@Test
+	public void deleteObjects() {
+		AmazonS3Client client = mock(AmazonS3Client.class);
+		S3Utils utils = new S3Utils(client);
+		List<KeyVersion> keys = new ArrayList<>();
+		for (int i = 0; i < 2000; i++) {
+			keys.add(new KeyVersion(""));
+		}
+
+		utils.delete("", keys);
+		
+		verify(client, times(2)).deleteObjects(any());
+	}
 }
