take another try at preventing memory issues.

Ben Osheroff · Ben Osheroff · commit 354bbec9ac2c · 2015-12-10T20:53:04.000-08:00
ObjectInputStream.readObject and ObjectOutputStream.writeObject do a
semi-pathological thing where they retain references to every object
that you've written into them (for efficiency purposes).  This means
that my whole premise about using them to flush ram out to disk was
flawed.
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellReplicator.java b/src/main/java/com/zendesk/maxwell/MaxwellReplicator.java
@@ -155,11 +155,11 @@ private RowMapBuffer getTransactionRows() throws Exception {
 
 		RowMapBuffer buffer = new RowMapBuffer(MAX_TX_ELEMENTS);
 
-		// currently to satisfy the test interface, the contract is to return null
-		// if the queue is empty.  should probably just replace this with an optional timeout...
-
 		while ( true ) {
 			v4Event = pollV4EventFromQueue();
+
+			// currently to satisfy the test interface, the contract is to return null
+			// if the queue is empty.  should probably just replace this with an optional timeout...
 			if (v4Event == null) {
 				ensureReplicatorThread();
 				continue;
diff --git a/src/main/java/com/zendesk/maxwell/util/ListWithDiskBuffer.java b/src/main/java/com/zendesk/maxwell/util/ListWithDiskBuffer.java
@@ -31,14 +31,19 @@ public void add(T element) throws IOException {
 			if ( file == null ) {
 				file = File.createTempFile("maxwell", "events");
 				file.deleteOnExit();
-				os = new ObjectOutputStream(new FileOutputStream(file));
-				is = new ObjectInputStream(new FileInputStream(file));
+				os = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
 			}
 
 			if ( elementsInFile == 0 )
 				LOGGER.debug("Overflowed in-memory buffer, spilling over into " + file);
 
-			os.writeObject(this.list.removeFirst());
+			os.writeUnshared(this.list.removeFirst());
+
+			if ( is == null && os != null ) {
+				os.flush();
+				is = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
+			}
+
 			elementsInFile++;
 		}
 	}
@@ -53,7 +58,7 @@ public T getLast() {
 
 	public T removeFirst() throws IOException, ClassNotFoundException {
 		if ( elementsInFile > 0 && is != null ) {
-			T element = (T) is.readObject();
+			T element = (T) is.readUnshared();
 			elementsInFile--;
 			return element;
 		} else {
