A few modifications/additions to the prior push (merged pull request). Added IsRespectUrlNamedAnchorOrHashbangEnabled as a config value and negated what the _removeUrlFragment logic. Added constructor to support the IsRespectUrlNamedAnchorOrHashbangEnabled on CsQueryHyperLinkParser.cs to have parity with HapHyperLinkParser.cs. Added a few unit tests to the HyperLinkParserTests.cs file.

sjdirect · sjdirect · commit eb719d8476d8 · 2015-09-19T14:10:58.000-07:00
diff --git a/Abot.Demo/App.config b/Abot.Demo/App.config
@@ -53,6 +53,7 @@
       isHttpRequestAutomaticDecompressionEnabled="false"
       isSendingCookiesEnabled="false"
       isSslCertificateValidationEnabled="false"
+      isRespectUrlNamedAnchorOrHashbangEnabled="false"
       minAvailableMemoryRequiredInMb="0"
       maxMemoryUsageInMb="0"
       maxMemoryUsageCacheTimeInSeconds="0"
diff --git a/Abot.Tests.Integration/App.config b/Abot.Tests.Integration/App.config
@@ -64,6 +64,7 @@
       isHttpRequestAutomaticDecompressionEnabled="false" 
       isSendingCookiesEnabled="false" 
       isSslCertificateValidationEnabled="false"
+      isRespectUrlNamedAnchorOrHashbangEnabled="false"
       minAvailableMemoryRequiredInMb="0" 
       maxMemoryUsageInMb="0" 
       maxMemoryUsageCacheTimeInSeconds="0" 
diff --git a/Abot.Tests.Unit/App.config b/Abot.Tests.Unit/App.config
@@ -63,7 +63,8 @@
       isHttpRequestAutoRedirectsEnabled="true" 
       isHttpRequestAutomaticDecompressionEnabled="true" 
       isSendingCookiesEnabled="true" 
-      isSslCertificateValidationEnabled="false" 
+      isSslCertificateValidationEnabled="false"
+      isRespectUrlNamedAnchorOrHashbangEnabled="true"
       minAvailableMemoryRequiredInMb="25" 
       maxMemoryUsageInMb="26" 
       maxMemoryUsageCacheTimeInSeconds="27" 
diff --git a/Abot.Tests.Unit/Core/AbotConfigurationSectionHandlerTest.cs b/Abot.Tests.Unit/Core/AbotConfigurationSectionHandlerTest.cs
@@ -30,6 +30,7 @@ public void GetSetion_FillsConfigValuesFromAppConfigFile()
             Assert.AreEqual(true, _uut.CrawlBehavior.IsHttpRequestAutomaticDecompressionEnabled);
             Assert.AreEqual(true, _uut.CrawlBehavior.IsSendingCookiesEnabled);
             Assert.AreEqual(false, _uut.CrawlBehavior.IsSslCertificateValidationEnabled);
+            Assert.AreEqual(true, _uut.CrawlBehavior.IsRespectUrlNamedAnchorOrHashbangEnabled);
             Assert.AreEqual(25, _uut.CrawlBehavior.MinAvailableMemoryRequiredInMb);
             Assert.AreEqual(26, _uut.CrawlBehavior.MaxMemoryUsageInMb);
             Assert.AreEqual(27, _uut.CrawlBehavior.MaxMemoryUsageCacheTimeInSeconds);
@@ -77,6 +78,7 @@ public void Convert_CovertsFromSectionObjectToDtoObject()
             Assert.AreEqual(true, _uut.CrawlBehavior.IsHttpRequestAutomaticDecompressionEnabled);
             Assert.AreEqual(true, _uut.CrawlBehavior.IsSendingCookiesEnabled);
             Assert.AreEqual(false, _uut.CrawlBehavior.IsSslCertificateValidationEnabled);
+            Assert.AreEqual(true, _uut.CrawlBehavior.IsRespectUrlNamedAnchorOrHashbangEnabled);
             Assert.AreEqual(result.MinAvailableMemoryRequiredInMb, _uut.CrawlBehavior.MinAvailableMemoryRequiredInMb);
             Assert.AreEqual(result.MaxMemoryUsageInMb, _uut.CrawlBehavior.MaxMemoryUsageInMb);
             Assert.AreEqual(result.MaxMemoryUsageCacheTimeInSeconds, _uut.CrawlBehavior.MaxMemoryUsageCacheTimeInSeconds);
diff --git a/Abot.Tests.Unit/Core/CsQueryHyperLinkParserTest.cs b/Abot.Tests.Unit/Core/CsQueryHyperLinkParserTest.cs
@@ -7,9 +7,9 @@ namespace Abot.Tests.Unit.Core
     [TestFixture]
     public class CsQueryHyperLinkParserTest : HyperLinkParserTest
     {
-        protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null)
+        protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
         {
-            return new CSQueryHyperlinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate);
+            return new CSQueryHyperlinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate, isRespectUrlNamedAnchorOrHashbangEnabled);
         }
 
         [Test]
diff --git a/Abot.Tests.Unit/Core/HapHyperLinkParserTest.cs b/Abot.Tests.Unit/Core/HapHyperLinkParserTest.cs
@@ -7,9 +7,9 @@ namespace Abot.Tests.Unit.Core
     [TestFixture]
     public class HapHyperLinkParserTest : HyperLinkParserTest
     {
-        protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null)
+        protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
         {
-            return new HapHyperLinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate);
+            return new HapHyperLinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate, isRespectUrlNamedAnchorOrHashbangEnabled);
         }
 
         [Test]
diff --git a/Abot.Tests.Unit/Core/HyperlinkParserTest.cs b/Abot.Tests.Unit/Core/HyperlinkParserTest.cs
@@ -16,13 +16,13 @@ public abstract class HyperLinkParserTest
         Uri _uri = new Uri("http://a.com/");
         CrawledPage _crawledPage;
 
-        protected abstract HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null);
+        protected abstract HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null, bool isRespectUrlNamedAnchorOrHashbangEnabled = false);
 
         [SetUp]
         public void Setup()
         {
             _crawledPage = new CrawledPage(_uri){ HttpWebRequest = (HttpWebRequest)WebRequest.Create(_uri) };
-            _unitUnderTest = GetInstance(false, false);
+            _unitUnderTest = GetInstance(false, false, null, false);
         }
 
         [Test]
@@ -189,14 +189,30 @@ public void GetLinks_DuplicateLinks_ReturnsOnlyOne()
         }
 
         [Test]
-        public void GetLinks_NamedAnchors_Ignores()
+        public void GetLinks_NamedAnchorsOrHashbangs_Ignores()
         {
-            _crawledPage.Content.Text =  "<a href=\"/aaa/a.html\" ></a><a href=\"/aaa/a.html#top\" ></a><a href=\"/aaa/a.html#bottom\" /></a>";
+            _crawledPage.Content.Text = "<a href=\"/aaa/a.html\" ></a><a href=\"/aaa/a.html#top\" ></a><a href=\"/aaa/a.html#bottom\" /></a><a href=\"/aaa/a.html/#someaction/someid\" /></a>";
 
             IEnumerable<Uri> result = _unitUnderTest.GetLinks(_crawledPage);
 
-            Assert.AreEqual(1, result.Count());
+            Assert.AreEqual(2, result.Count());
+            Assert.AreEqual("http://a.com/aaa/a.html", result.ElementAt(0).AbsoluteUri);
+            Assert.AreEqual("http://a.com/aaa/a.html/", result.ElementAt(1).AbsoluteUri);
+        }
+
+        [Test]
+        public void GetLinks_NamedAnchorsOrHashbangs_Enabled_ReturnsLinks()
+        {
+            _unitUnderTest = GetInstance(false, false, null, true);
+            _crawledPage.Content.Text = "<a href=\"/aaa/a.html\" ></a><a href=\"/aaa/a.html#top\" ></a><a href=\"/aaa/a.html#bottom\" /></a><a href=\"/aaa/a.html/#someaction/someid\" /></a>";
+
+            IEnumerable<Uri> result = _unitUnderTest.GetLinks(_crawledPage);
+
+            Assert.AreEqual(4, result.Count());
             Assert.AreEqual("http://a.com/aaa/a.html", result.ElementAt(0).AbsoluteUri);
+            Assert.AreEqual("http://a.com/aaa/a.html#top", result.ElementAt(1).AbsoluteUri);
+            Assert.AreEqual("http://a.com/aaa/a.html#bottom", result.ElementAt(2).AbsoluteUri);
+            Assert.AreEqual("http://a.com/aaa/a.html/#someaction/someid", result.ElementAt(3).AbsoluteUri);
         }
 
         [Test]
diff --git a/Abot.Tests.Unit/Poco/CrawlConfigurationTest.cs b/Abot.Tests.Unit/Poco/CrawlConfigurationTest.cs
@@ -36,6 +36,7 @@ public void Constructor_ValidUri_CreatesInstance()
             Assert.AreEqual(false, unitUnderTest.IsHttpRequestAutomaticDecompressionEnabled);
             Assert.AreEqual(false, unitUnderTest.IsSendingCookiesEnabled);
             Assert.AreEqual(true, unitUnderTest.IsSslCertificateValidationEnabled);
+            Assert.AreEqual(false, unitUnderTest.IsRespectUrlNamedAnchorOrHashbangEnabled);
             Assert.AreEqual(0, unitUnderTest.MaxMemoryUsageCacheTimeInSeconds);
             Assert.AreEqual(0, unitUnderTest.MaxMemoryUsageInMb);
             Assert.AreEqual(0, unitUnderTest.MinAvailableMemoryRequiredInMb);
diff --git a/Abot/Core/AbotConfigurationSectionHandler.cs b/Abot/Core/AbotConfigurationSectionHandler.cs
@@ -242,6 +242,12 @@ public bool IsSendingCookiesEnabled
             get { return (bool)this["isSendingCookiesEnabled"]; }
         }
 
+        [ConfigurationProperty("isRespectUrlNamedAnchorOrHashbangEnabled", IsRequired = false)]
+        public bool IsRespectUrlNamedAnchorOrHashbangEnabled
+        {
+            get { return (bool)this["isRespectUrlNamedAnchorOrHashbangEnabled"]; }
+        }
+
         [ConfigurationProperty("minAvailableMemoryRequiredInMb", IsRequired = false)]
         public int MinAvailableMemoryRequiredInMb
         {
diff --git a/Abot/Core/CsQueryHyperLinkParser.cs b/Abot/Core/CsQueryHyperLinkParser.cs
@@ -19,8 +19,18 @@ public CSQueryHyperlinkParser()
         {
         }
 
-        public CSQueryHyperlinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanURLFunc = null)
-            : base(isRespectMetaRobotsNoFollowEnabled)
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="isRespectMetaRobotsNoFollowEnabled">Whether parser should ignore pages with meta no robots</param>
+        /// <param name="isRespectAnchorRelNoFollowEnabled">Whether parser should ignore links with rel no follow</param>
+        /// <param name="cleanURLFunc">Function to clean the url</param>
+        /// <param name="isRespectUrlNamedAnchorOrHashbangEnabled">Whether parser should consider named anchor and/or hashbang '#' character as part of the url</param>
+        public CSQueryHyperlinkParser(  bool isRespectMetaRobotsNoFollowEnabled, 
+                                        bool isRespectAnchorRelNoFollowEnabled, 
+                                        Func<string, string> cleanURLFunc = null, 
+                                        bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
+            : base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled)
         {
             _isRespectAnchorRelNoFollowEnabled = isRespectAnchorRelNoFollowEnabled;
             _cleanURLFunc = cleanURLFunc;
diff --git a/Abot/Core/HapHyperLinkParser.cs b/Abot/Core/HapHyperLinkParser.cs
@@ -25,11 +25,18 @@ public HapHyperLinkParser()
         {
         }
 
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="isRespectMetaRobotsNoFollowEnabled">Whether parser should ignore pages with meta no robots</param>
+        /// <param name="isRespectAnchorRelNoFollowEnabled">Whether parser should ignore links with rel no follow</param>
+        /// <param name="cleanURLFunc">Function to clean the url</param>
+        /// <param name="isRespectUrlNamedAnchorOrHashbangEnabled">Whether parser should consider named anchor and/or hashbang '#' character as part of the url</param>
         public HapHyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled,
                                   bool isRespectAnchorRelNoFollowEnabled,
                                   Func<string, string> cleanURLFunc = null,
-                                  bool removeUrlFragment = true)
-            :base(isRespectMetaRobotsNoFollowEnabled, removeUrlFragment)
+                                  bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
+            :base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled)
         {
             _isRespectAnchorRelNoFollowEnabled = isRespectAnchorRelNoFollowEnabled;
             _cleanURLFunc = cleanURLFunc;
diff --git a/Abot/Core/HyperLinkParser.cs b/Abot/Core/HyperLinkParser.cs
@@ -23,18 +23,18 @@ public abstract class HyperLinkParser : IHyperLinkParser
     {
         protected ILog _logger = LogManager.GetLogger("AbotLogger");
         protected bool _isRespectMetaRobotsNoFollowEnabled;
-        protected bool _removeUrlFragment;
+        protected bool _isRespectUrlNamedAnchorOrHashbangEnabled; 
 
         public HyperLinkParser()
             :this(false, true)
         {
 
         }
 
-        public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool removeUrlFragment = true)
+        public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
         {
             _isRespectMetaRobotsNoFollowEnabled = isRespectMetaRobotsNoFollowEnabled;
-            _removeUrlFragment = removeUrlFragment;
+            _isRespectUrlNamedAnchorOrHashbangEnabled = isRespectUrlNamedAnchorOrHashbangEnabled;
         }
 
         /// <summary>
@@ -100,9 +100,9 @@ protected virtual List<Uri> GetUris(CrawledPage crawledPage, IEnumerable<string>
                 {
                     // Remove the url fragment part of the url if needed.
                     // This is the part after the # and is often not useful.
-                    href = _removeUrlFragment
-                        ? hrefValue.Split('#')[0]
-                        : hrefValue;
+                    href = _isRespectUrlNamedAnchorOrHashbangEnabled
+                        ? hrefValue
+                        : hrefValue.Split('#')[0];
                     Uri newUri = new Uri(uriToUse, href);
 
                     if (!uris.Exists(u => u.AbsoluteUri == newUri.AbsoluteUri))
diff --git a/Abot/Crawler/WebCrawler.cs b/Abot/Crawler/WebCrawler.cs
@@ -198,7 +198,7 @@ public WebCrawler(
                 || _crawlContext.CrawlConfiguration.MinAvailableMemoryRequiredInMb > 0)
                 _memoryManager = memoryManager ?? new MemoryManager(new CachedMemoryMonitor(new GcMemoryMonitor(), _crawlContext.CrawlConfiguration.MaxMemoryUsageCacheTimeInSeconds));
 
-            _hyperLinkParser = hyperLinkParser ?? new HapHyperLinkParser(_crawlContext.CrawlConfiguration.IsRespectMetaRobotsNoFollowEnabled, _crawlContext.CrawlConfiguration.IsRespectAnchorRelNoFollowEnabled);
+            _hyperLinkParser = hyperLinkParser ?? new HapHyperLinkParser(_crawlContext.CrawlConfiguration.IsRespectMetaRobotsNoFollowEnabled, _crawlContext.CrawlConfiguration.IsRespectAnchorRelNoFollowEnabled, null, _crawlContext.CrawlConfiguration.IsRespectUrlNamedAnchorOrHashbangEnabled);
 
             _crawlContext.Scheduler = _scheduler;
         }
diff --git a/Abot/Poco/CrawlConfiguration.cs b/Abot/Poco/CrawlConfiguration.cs
@@ -86,6 +86,11 @@ public CrawlConfiguration()
         /// </summary>
         public bool IsExternalPageLinksCrawlingEnabled { get; set; }
 
+        /// <summary>
+        /// Whether or not url named anchors or hashbangs are considered part of the url. If false, they will be ignored. If true, they will be considered part of the url.
+        /// </summary>
+        public bool IsRespectUrlNamedAnchorOrHashbangEnabled { get; set; }
+
         /// <summary>
         /// A comma seperated string that has content types that should have their page content downloaded. For each page, the content type is checked to see if it contains any of the values defined here.
         /// </summary>

Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,9 @@ namespace Abot.Tests.Unit.Core`
`7`	`7`	`[TestFixture]`
`8`	`8`	`public class CsQueryHyperLinkParserTest : HyperLinkParserTest`
`9`	`9`	`{`
`10`		`- protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null)`
	`10`	`+ protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)`
`11`	`11`	`{`
`12`		`- return new CSQueryHyperlinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate);`
	`12`	`+ return new CSQueryHyperlinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate, isRespectUrlNamedAnchorOrHashbangEnabled);`
`13`	`13`	`}`
`14`	`14`
`15`	`15`	`[Test]`
Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,9 @@ namespace Abot.Tests.Unit.Core`
`7`	`7`	`[TestFixture]`
`8`	`8`	`public class HapHyperLinkParserTest : HyperLinkParserTest`
`9`	`9`	`{`
`10`		`- protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null)`
	`10`	`+ protected override HyperLinkParser GetInstance(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectAnchorRelNoFollowEnabled, Func<string, string> cleanUrlDelegate = null, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)`
`11`	`11`	`{`
`12`		`- return new HapHyperLinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate);`
	`12`	`+ return new HapHyperLinkParser(isRespectMetaRobotsNoFollowEnabled, isRespectAnchorRelNoFollowEnabled, cleanUrlDelegate, isRespectUrlNamedAnchorOrHashbangEnabled);`
`13`	`13`	`}`
`14`	`14`
`15`	`15`	`[Test]`
Original file line number	Diff line number	Diff line change
`@@ -242,6 +242,12 @@ public bool IsSendingCookiesEnabled`
`242`	`242`	`get { return (bool)this["isSendingCookiesEnabled"]; }`
`243`	`243`	`}`
`244`	`244`
	`245`	`+ [ConfigurationProperty("isRespectUrlNamedAnchorOrHashbangEnabled", IsRequired = false)]`
	`246`	`+ public bool IsRespectUrlNamedAnchorOrHashbangEnabled`
	`247`	`+ {`
	`248`	`+ get { return (bool)this["isRespectUrlNamedAnchorOrHashbangEnabled"]; }`
	`249`	`+ }`
	`250`	`+`
`245`	`251`	`[ConfigurationProperty("minAvailableMemoryRequiredInMb", IsRequired = false)]`
`246`	`252`	`public int MinAvailableMemoryRequiredInMb`
`247`	`253`	`{`