Only group elements ten at a time into SequenceFile records in

mateiz · mateiz · commit 5a7b3702253c · 2012-10-04T16:49:30.000-07:00
saveAsObjectFile
diff --git a/core/src/main/scala/spark/RDD.scala b/core/src/main/scala/spark/RDD.scala
@@ -256,7 +256,7 @@ abstract class RDD[T: ClassManifest](@transient sc: SparkContext) extends Serial
   }
 
   def saveAsObjectFile(path: String) {
-    this.glom
+    this.mapPartitions(iter => iter.grouped(10).map(_.toArray))
       .map(x => (NullWritable.get(), new BytesWritable(Utils.serialize(x))))
       .saveAsSequenceFile(path)
   }

Original file line number	Diff line number	Diff line change
`@@ -256,7 +256,7 @@ abstract class RDD[T: ClassManifest](@transient sc: SparkContext) extends Serial`
`256`	`256`	`}`
`257`	`257`
`258`	`258`	`def saveAsObjectFile(path: String) {`
`259`		`- this.glom`
	`259`	`+ this.mapPartitions(iter => iter.grouped(10).map(_.toArray))`
`260`	`260`	`.map(x => (NullWritable.get(), new BytesWritable(Utils.serialize(x))))`
`261`	`261`	`.saveAsSequenceFile(path)`
`262`	`262`	`}`