Merge branch 'master' of github.com:sjdirect/abot

sjdirect · sjdirect · commit 56b75d47451c · 2015-11-02T12:33:33.000-07:00
diff --git a/Abot/Core/PageRequester.cs b/Abot/Core/PageRequester.cs
@@ -6,6 +6,7 @@
 using System.Net;
 using System.Reflection;
 using System.Threading.Tasks;
+using log4net.Core;
 
 namespace Abot.Core
 {
@@ -103,24 +104,32 @@ public virtual CrawledPage MakeRequest(Uri uri, Func<CrawledPage, CrawlDecision>
             }
             finally
             {
-                crawledPage.HttpWebRequest = request;
-                crawledPage.RequestCompleted = DateTime.Now;
-                if (response != null)
+                try
                 {
-                    crawledPage.HttpWebResponse = new HttpWebResponseWrapper(response);
-                    CrawlDecision shouldDownloadContentDecision = shouldDownloadContent(crawledPage);
-                    if (shouldDownloadContentDecision.Allow)
+                    crawledPage.HttpWebRequest = request;
+                    crawledPage.RequestCompleted = DateTime.Now;
+                    if (response != null)
                     {
-                        crawledPage.DownloadContentStarted = DateTime.Now;
-                        crawledPage.Content = _extractor.GetContent(response);
-                        crawledPage.DownloadContentCompleted = DateTime.Now;
+                        crawledPage.HttpWebResponse = new HttpWebResponseWrapper(response);
+                        CrawlDecision shouldDownloadContentDecision = shouldDownloadContent(crawledPage);
+                        if (shouldDownloadContentDecision.Allow)
+                        {
+                            crawledPage.DownloadContentStarted = DateTime.Now;
+                            crawledPage.Content = _extractor.GetContent(response);
+                            crawledPage.DownloadContentCompleted = DateTime.Now;
+                        }
+                        else
+                        {
+                            _logger.DebugFormat("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldDownloadContentDecision.Reason);
+                        }
+
+                        response.Close();//Should already be closed by _extractor but just being safe
                     }
-                    else
-                    {
-                        _logger.DebugFormat("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldDownloadContentDecision.Reason);
-                    }
-
-                    response.Close();//Should already be closed by _extractor but just being safe
+                }
+                catch (Exception e)
+                {
+                    _logger.DebugFormat("Error occurred finalizing requesting url [{0}]", uri.AbsoluteUri);
+                    _logger.Debug(e);
                 }
             }
 
diff --git a/Abot/Poco/HttpWebResponseWrapper.cs b/Abot/Poco/HttpWebResponseWrapper.cs
@@ -39,14 +39,26 @@ public HttpWebResponseWrapper(HttpWebResponse response)
             this.Cookies = response.Cookies;
             this.IsFromCache = response.IsFromCache;
             this.IsMutuallyAuthenticated = response.IsMutuallyAuthenticated;
-            this.LastModified = response.LastModified;
+            this.LastModified = GetLastModified(response);
             this.Method = response.Method;
             this.ProtocolVersion = response.ProtocolVersion;
             this.ResponseUri = response.ResponseUri;
             this.Server = response.Server;
             this.StatusDescription = response.StatusDescription;
         }
 
+        private static DateTime GetLastModified(HttpWebResponse response)
+        {
+            try
+            {
+                return response.LastModified;
+            }
+            catch (ProtocolViolationException)
+            {
+                return DateTime.MinValue;
+            }
+        }
+
         /// <summary>Constructs a response based on custom parameters.</summary>
         /// <remarks>Recieves parameters neccesarily set for Abot to work.</remarks>
         public HttpWebResponseWrapper(HttpStatusCode statusCode, string contentType, byte[] content, NameValueCollection headers)
diff --git a/README.md b/README.md
@@ -287,7 +287,7 @@ crawler.ShouldDownloadPageContent((crawledPage, crawlContext) =>
 crawler.ShouldCrawlPageLinks((crawledPage, crawlContext) =>
 {
 	CrawlDecision decision = new CrawlDecision{ Allow = true };
-	if (crawledPage.PageSizeInBytes < 100)
+	if (crawledPage.Content.Bytes.Length < 100)
 		return new CrawlDecision { Allow = false, Reason = "Just crawl links in pages that have at least 100 bytes" };
 
 	return decision;

Original file line number	Diff line number	Diff line change
`@@ -287,7 +287,7 @@ crawler.ShouldDownloadPageContent((crawledPage, crawlContext) =>`
`287`	`287`	`crawler.ShouldCrawlPageLinks((crawledPage, crawlContext) =>`
`288`	`288`	`{`
`289`	`289`	`CrawlDecision decision = new CrawlDecision{ Allow = true };`
`290`		`- if (crawledPage.PageSizeInBytes < 100)`
	`290`	`+ if (crawledPage.Content.Bytes.Length < 100)`
`291`	`291`	`return new CrawlDecision { Allow = false, Reason = "Just crawl links in pages that have at least 100 bytes" };`
`292`	`292`
`293`	`293`	`return decision;`