Closes n1k0#10 - Added console.log messages to JSON result + UI.

n1k0 · n1k0 · commit 04cd62dec7fa · 2015-03-17T22:16:26.000+01:00
diff --git a/phantom-scrape.js b/phantom-scrape.js
@@ -3,6 +3,7 @@ var page = require("webpage").create();
 var url = system.args[1];
 var readabilityPath = system.args[2];
 var userAgent = system.args[3];
+var consoleLogs = [];
 
 // Prevent page js errors to break JSON output
 // XXX: should we log these instead?
@@ -17,6 +18,43 @@ function outputJSON(object) {
   console.log(JSON.stringify(object, null, 2));
 }
 
+/**
+ * Note: This function runs within page environment.
+ */
+function runReadability(url, userAgent, pageContent) {
+  var location = document.location;
+  var uri = {
+    spec: location.href,
+    host: location.host,
+    prePath: location.protocol + "//" + location.host, // TODO This is incomplete, needs username/password and port
+    scheme: location.protocol.substr(0, location.protocol.indexOf(":")),
+    pathBase: location.protocol + "//" + location.host + location.pathname.substr(0, location.pathname.lastIndexOf("/") + 1)
+  };
+  try {
+    var result = new Readability(uri, document).parse();
+    if (result) {
+      result.userAgent = userAgent;
+    } else {
+      result = {
+        error: {
+          message: "Empty result from Readability.js.",
+          sourceHTML: pageContent || "Empty page content."
+        }
+      };
+    }
+    return result;
+  } catch (err) {
+    return {
+      error: {
+        message: err.message,
+        line: err.line,
+        stack: err.stack,
+        sourceHTML: pageContent || "Empty page content."
+      }
+    };
+  }
+};
+
 if (!url) {
   exitWithError("Missing url arg.");
 } else if (!readabilityPath) {
@@ -33,45 +71,23 @@ page.settings.loadImages = false;
 // ensure we don't waste time trying to load slow/missing resources
 page.settings.resourceTimeout = 1000;
 
+page.onConsoleMessage = function(msg) {
+  consoleLogs.push(msg);
+};
+
 page.open(url, function(status) {
   if (status !== "success") {
     return exitWithError("Unable to access " + url);
   }
   if (!page.injectJs(readabilityPath)) {
     exitWithError("Couldn't inject " + readabilityPath);
   }
-  outputJSON(page.evaluate(function(url, userAgent, pageContent) {
-    var location = document.location;
-    var uri = {
-      spec: location.href,
-      host: location.host,
-      prePath: location.protocol + "//" + location.host, // TODO This is incomplete, needs username/password and port
-      scheme: location.protocol.substr(0, location.protocol.indexOf(":")),
-      pathBase: location.protocol + "//" + location.host + location.pathname.substr(0, location.pathname.lastIndexOf("/") + 1)
-    };
-    try {
-      var result = new Readability(uri, document).parse();
-      if (result) {
-        result.userAgent = userAgent;
-      } else {
-        result = {
-          error: {
-            message: "Empty result from Readability.js.",
-            sourceHTML: pageContent || "Empty page content."
-          }
-        };
-      }
-      return result;
-    } catch (err) {
-      return {
-        error: {
-          message: err.message,
-          line: err.line,
-          stack: err.stack,
-          sourceHTML: pageContent || "Empty page content."
-        }
-      };
-    }
-  }, url, page.settings.userAgent, page.content));
+  var result = page.evaluate(runReadability, url, page.settings.userAgent, page.content);
+  if (result && result.error) {
+    result.error.consoleLogs = consoleLogs;
+  } else if (result && result.content) {
+    result.consoleLogs = consoleLogs;
+  }
+  outputJSON(result);
   phantom.exit();
 });
diff --git a/scrape.js b/scrape.js
@@ -35,6 +35,8 @@ module.exports = function scrape(url, options) {
       }
       if (error) {
         reject(objectAssign(new Error(), error));
+      } else if (!response) {
+        reject(new Error("Empty scraped response."));
       } else {
         fulfill(response);
       }
diff --git a/server.js b/server.js
@@ -46,7 +46,7 @@ app.get("/api/get", function(req, res) {
     res.json(sanitize ? sanitizeResult(result) : result);
   }).catch(function(err) {
     console.log(err);
-    res.status(500).json({error: JSON.parse(JSON.stringify(err))});
+    res.status(500).json({error: {message: err.message}});
   });
 });
 
diff --git a/static/index.html b/static/index.html
@@ -5,6 +5,13 @@
   <title>Readability.js test page</title>
   <link rel="stylesheet" type="text/css" href="bootstrap.min.css">
   <link rel="stylesheet" type="text/css" href="style.css">
+  <style>
+    #logs {
+      width: 100%;
+      height: 180px;
+      font-family: monospace;
+    }
+  </style>
 </head>
 <body>
   <div class="container-fluid">
@@ -65,6 +72,10 @@ <h1>Readability.js <small>test page</small></h1>
         </div>
       </div>
     </div>
+    <div>
+      <h3>Console logs</h3>
+      <textarea id="logs"></textarea>
+    </div>
   </div>
   <script src="main.js"></script>
 </body>
diff --git a/static/main.js b/static/main.js
@@ -22,6 +22,7 @@
       q("#length").textContent = "";
       q("#dir").textContent = "";
       q("#excerpt").textContent = "";
+      q("#logs").value = "";
       target.contentDocument.body.innerHTML = "";
     } else {
       q("#error").textContent = "";
@@ -30,6 +31,7 @@
       q("#length").textContent = jsonResponse.length;
       q("#dir").textContent = jsonResponse.dir;
       q("#excerpt").textContent = jsonResponse.excerpt;
+      q("#logs").value = (jsonResponse.consoleLogs || []).join("\n");
       target.contentDocument.body.innerHTML = jsonResponse.content;
     }
   }

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,8 @@ module.exports = function scrape(url, options) {`
`35`	`35`	`}`
`36`	`36`	`if (error) {`
`37`	`37`	`reject(objectAssign(new Error(), error));`
	`38`	`+ } else if (!response) {`
	`39`	`+ reject(new Error("Empty scraped response."));`
`38`	`40`	`} else {`
`39`	`41`	`fulfill(response);`
`40`	`42`	`}`