Have added batched statements so it's possible to process large result sets without running out of memory

Bennett Thompson · Bennett Thompson · commit c2f838c9e09a · 2016-10-25T13:12:56.000+13:00
diff --git a/lib/triagens/ArangoDb/BatchedCursor.php b/lib/triagens/ArangoDb/BatchedCursor.php
@@ -0,0 +1,48 @@
+<?php
+namespace triagens\ArangoDb;
+class BatchedCursor extends Cursor
+{
+
+    private $_processed = 0;
+    public function setResults( array $data )
+    {
+        $this->_result=[];
+        $this->add( $data );
+        $this->rewind();
+        $this->updateLength();
+    }
+
+    public function fetchNextBatch()
+    {
+        // continuation
+        $this->_processed += $this->_length;
+        $response = $this->_connection->put($this->url() . '/' . $this->_id, '', array());
+        ++$this->_fetches;
+
+        $data = $response->getJson();
+
+        $this->_hasMore = (bool) $data[Cursor::ENTRY_HASMORE];
+        $this->setResults($data[Cursor::ENTRY_RESULT]);
+
+        if (!$this->_hasMore) {
+            // we have fetched the complete result set and can unset the id now
+            $this->_id = null;
+        }
+        return $this->_result;
+    }
+
+    public function next()
+    {
+        parent::next();
+        if( $this->key() >= $this->_length && $this->_hasMore )
+        {
+            $this->fetchNextBatch();
+        }
+    }
+
+    public function fullKey()
+    {
+        return $this->_processed + $this->key();
+    }
+}
+?>
diff --git a/lib/triagens/ArangoDb/BatchedStatement.php b/lib/triagens/ArangoDb/BatchedStatement.php
@@ -0,0 +1,16 @@
+<?php
+/**
+ * Created by PhpStorm.
+ * User: Bennett.Thompson
+ * Date: 25/10/2016
+ * Time: 11:48 AM
+ */
+
+namespace triagens\ArangoDb;
+class BatchedStatement extends Statement
+{
+    protected static function getCursor( $connection, $json, $options )
+    {
+        return new BatchedCursor( $connection, $json, $options );
+    }
+}
diff --git a/lib/triagens/ArangoDb/Cursor.php b/lib/triagens/ArangoDb/Cursor.php
@@ -31,7 +31,7 @@ class Cursor implements
      *
      * @var Connection
      */
-    private $_connection;
+    protected $_connection;
     /**
      * Cursor options
      *
@@ -51,21 +51,21 @@ class Cursor implements
      *
      * @var array
      */
-    private $_result;
+    protected $_result;
 
     /**
      * "has more" indicator - if true, the server has more results
      *
      * @var bool
      */
-    private $_hasMore;
+    protected $_hasMore;
 
     /**
      * cursor id - might be NULL if cursor does not have an id
      *
      * @var mixed
      */
-    private $_id;
+    protected $_id;
 
     /**
      * current position in result set iteration (zero-based)
@@ -79,7 +79,7 @@ class Cursor implements
      *
      * @var int
      */
-    private $_length;
+    protected $_length;
 
     /**
      * full count of the result set (ignoring the outermost LIMIT)
@@ -98,7 +98,7 @@ class Cursor implements
     /**
      * number of HTTP calls that were made to build the cursor result
      */
-    private $_fetches = 1;
+    protected $_fetches = 1;
 
     /**
      * whether or not the query result was served from the AQL query result cache
@@ -378,7 +378,7 @@ public function valid()
      *
      * @return void
      */
-    private function add(array $data)
+    protected function add(array $data)
     {
         foreach ($this->sanitize($data) as $row) {
             if (!is_array($row) || (isset($this->_options[self::ENTRY_FLAT]) && $this->_options[self::ENTRY_FLAT])) {
@@ -672,7 +672,7 @@ private function fetchOutstanding()
      *
      * @return void
      */
-    private function updateLength()
+    protected function updateLength()
     {
         $this->_length = count($this->_result);
     }
@@ -683,7 +683,7 @@ private function updateLength()
      *
      * @return string
      */
-    private function url()
+    protected function url()
     {
         if (isset($this->_options[self::ENTRY_BASEURL])) {
             return $this->_options[self::ENTRY_BASEURL];
diff --git a/lib/triagens/ArangoDb/Statement.php b/lib/triagens/ArangoDb/Statement.php
@@ -250,7 +250,7 @@ public function execute()
         while (true) {
             try {
                 $response = $this->_connection->post(Urls::URL_CURSOR, $this->getConnection()->json_encode_wrapper($data), array());
-                return new Cursor($this->_connection, $response->getJson(), $this->getCursorOptions());
+                return static::getCursor($this->_connection, $response->getJson(), $this->getCursorOptions());
             } catch (ServerException $e) {
                 if ($tries++ >= $this->_retries) {
                     throw $e;
@@ -264,6 +264,10 @@ public function execute()
         }
     }
 
+    protected static function getCursor($connection, $json, $options)
+    {
+        return new Cursor( $connection, $json, $options );
+    }
 
     /**
      * Explain the statement's execution plan

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ class Cursor implements`
`31`	`31`	`*`
`32`	`32`	`* @var Connection`
`33`	`33`	`*/`
`34`		`- private $_connection;`
	`34`	`+ protected $_connection;`
`35`	`35`	`/**`
`36`	`36`	`* Cursor options`
`37`	`37`	`*`
`@@ -51,21 +51,21 @@ class Cursor implements`
`51`	`51`	`*`
`52`	`52`	`* @var array`
`53`	`53`	`*/`
`54`		`- private $_result;`
	`54`	`+ protected $_result;`
`55`	`55`
`56`	`56`	`/**`
`57`	`57`	`* "has more" indicator - if true, the server has more results`
`58`	`58`	`*`
`59`	`59`	`* @var bool`
`60`	`60`	`*/`
`61`		`- private $_hasMore;`
	`61`	`+ protected $_hasMore;`
`62`	`62`
`63`	`63`	`/**`
`64`	`64`	`* cursor id - might be NULL if cursor does not have an id`
`65`	`65`	`*`
`66`	`66`	`* @var mixed`
`67`	`67`	`*/`
`68`		`- private $_id;`
	`68`	`+ protected $_id;`
`69`	`69`
`70`	`70`	`/**`
`71`	`71`	`* current position in result set iteration (zero-based)`
`@@ -79,7 +79,7 @@ class Cursor implements`
`79`	`79`	`*`
`80`	`80`	`* @var int`
`81`	`81`	`*/`
`82`		`- private $_length;`
	`82`	`+ protected $_length;`
`83`	`83`
`84`	`84`	`/**`
`85`	`85`	`* full count of the result set (ignoring the outermost LIMIT)`
`@@ -98,7 +98,7 @@ class Cursor implements`
`98`	`98`	`/**`
`99`	`99`	`* number of HTTP calls that were made to build the cursor result`
`100`	`100`	`*/`
`101`		`- private $_fetches = 1;`
	`101`	`+ protected $_fetches = 1;`
`102`	`102`
`103`	`103`	`/**`
`104`	`104`	`* whether or not the query result was served from the AQL query result cache`
`@@ -378,7 +378,7 @@ public function valid()`
`378`	`378`	`*`
`379`	`379`	`* @return void`
`380`	`380`	`*/`
`381`		`- private function add(array $data)`
	`381`	`+ protected function add(array $data)`
`382`	`382`	`{`
`383`	`383`	`foreach ($this->sanitize($data) as $row) {`
`384`	`384`	`if (!is_array($row) \|\| (isset($this->_options[self::ENTRY_FLAT]) && $this->_options[self::ENTRY_FLAT])) {`
`@@ -672,7 +672,7 @@ private function fetchOutstanding()`
`672`	`672`	`*`
`673`	`673`	`* @return void`
`674`	`674`	`*/`
`675`		`- private function updateLength()`
	`675`	`+ protected function updateLength()`
`676`	`676`	`{`
`677`	`677`	`$this->_length = count($this->_result);`
`678`	`678`	`}`
`@@ -683,7 +683,7 @@ private function updateLength()`
`683`	`683`	`*`
`684`	`684`	`* @return string`
`685`	`685`	`*/`
`686`		`- private function url()`
	`686`	`+ protected function url()`
`687`	`687`	`{`
`688`	`688`	`if (isset($this->_options[self::ENTRY_BASEURL])) {`
`689`	`689`	`return $this->_options[self::ENTRY_BASEURL];`