Moved the cleanURLFun to the base class where it can clean on the full absolute link, fixes sjdirect#101

sjdirect · sjdirect · commit 58eb440185fd · 2015-12-05T09:50:52.000-07:00
diff --git a/Abot.Tests.Unit/Core/HyperlinkParserTest.cs b/Abot.Tests.Unit/Core/HyperlinkParserTest.cs
@@ -427,8 +427,8 @@ public void GetLinks_CleanUrlDelegateSet_ReturnsCleanLinks()
 
             Assert.IsNotNull(result);
             Assert.AreEqual(2, result.Count());
-            Assert.AreEqual("http://a.com/xxx/x.html", result.ElementAt(0).AbsoluteUri);
-            Assert.AreEqual("http://a.com/yyy/y.html", result.ElementAt(1).AbsoluteUri);
+            Assert.AreEqual("http://x.com/xxx/x.html", result.ElementAt(0).AbsoluteUri);
+            Assert.AreEqual("http://x.com/yyy/y.html", result.ElementAt(1).AbsoluteUri);
         }
 
         [Test] //https://github.com/sjdirect/abot/issues/15
diff --git a/Abot/Core/CsQueryHyperLinkParser.cs b/Abot/Core/CsQueryHyperLinkParser.cs
@@ -12,7 +12,6 @@ namespace Abot.Core
     [Serializable]
     public class CSQueryHyperlinkParser : HyperLinkParser
     {
-        Func<string, string> _cleanURLFunc;
         bool _isRespectAnchorRelNoFollowEnabled;
         
         public CSQueryHyperlinkParser()
@@ -30,10 +29,9 @@ public CSQueryHyperlinkParser(  bool isRespectMetaRobotsNoFollowEnabled,
                                         bool isRespectAnchorRelNoFollowEnabled, 
                                         Func<string, string> cleanURLFunc = null, 
                                         bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
-            : base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled)
+            : base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled, cleanURLFunc)
         {
             _isRespectAnchorRelNoFollowEnabled = isRespectAnchorRelNoFollowEnabled;
-            _cleanURLFunc = cleanURLFunc;
         }
         
         protected override string ParserType
@@ -49,7 +47,7 @@ protected override IEnumerable<string> GetHrefValues(CrawledPage crawledPage)
             IEnumerable<string> hrefValues = crawledPage.CsQueryDocument.Select("a, area")
             .Elements
             .Where(e => !HasRelNoFollow(e))
-            .Select(y => _cleanURLFunc != null ? _cleanURLFunc(y.GetAttribute("href")) : y.GetAttribute("href"))
+            .Select(y => y.GetAttribute("href"))
             .Where(a => !string.IsNullOrWhiteSpace(a));
 
             IEnumerable<string> canonicalHref = crawledPage.CsQueryDocument.
diff --git a/Abot/Core/HapHyperLinkParser.cs b/Abot/Core/HapHyperLinkParser.cs
@@ -12,7 +12,6 @@ namespace Abot.Core
     [Serializable]
     public class HapHyperLinkParser : HyperLinkParser
     {
-        Func<string, string> _cleanURLFunc;
         bool _isRespectAnchorRelNoFollowEnabled;
 
         protected override string ParserType
@@ -36,10 +35,9 @@ public HapHyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled,
                                   bool isRespectAnchorRelNoFollowEnabled,
                                   Func<string, string> cleanURLFunc = null,
                                   bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
-            :base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled)
+            :base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled, cleanURLFunc)
         {
             _isRespectAnchorRelNoFollowEnabled = isRespectAnchorRelNoFollowEnabled;
-            _cleanURLFunc = cleanURLFunc;
         }
 
         protected override IEnumerable<string> GetHrefValues(CrawledPage crawledPage)
@@ -94,7 +92,7 @@ protected virtual List<string> GetLinks(HtmlNodeCollection nodes)
                 if (HasRelNoFollow(node))
                     continue;
 
-                hrefValue = _cleanURLFunc != null ? _cleanURLFunc(node.Attributes["href"].Value) : node.Attributes["href"].Value;
+                hrefValue = node.Attributes["href"].Value;
                 if (!string.IsNullOrWhiteSpace(hrefValue))
                 {
                     hrefValue = DeEntitize(hrefValue);
diff --git a/Abot/Core/HyperLinkParser.cs b/Abot/Core/HyperLinkParser.cs
@@ -23,18 +23,20 @@ public abstract class HyperLinkParser : IHyperLinkParser
     {
         protected ILog _logger = LogManager.GetLogger("AbotLogger");
         protected bool _isRespectMetaRobotsNoFollowEnabled;
-        protected bool _isRespectUrlNamedAnchorOrHashbangEnabled; 
+        protected bool _isRespectUrlNamedAnchorOrHashbangEnabled;
+        protected Func<string, string> _cleanURLFunc;
 
         public HyperLinkParser()
-            :this(false, true)
+            :this(false, true, null)
         {
 
         }
 
-        public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)
+        public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectUrlNamedAnchorOrHashbangEnabled, Func<string, string> cleanURLFunc)
         {
             _isRespectMetaRobotsNoFollowEnabled = isRespectMetaRobotsNoFollowEnabled;
             _isRespectUrlNamedAnchorOrHashbangEnabled = isRespectUrlNamedAnchorOrHashbangEnabled;
+            _cleanURLFunc = cleanURLFunc;
         }
 
         /// <summary>
@@ -105,6 +107,9 @@ protected virtual List<Uri> GetUris(CrawledPage crawledPage, IEnumerable<string>
                         : hrefValue.Split('#')[0];
                     Uri newUri = new Uri(uriToUse, href);
 
+                    if (_cleanURLFunc != null)
+                        newUri = new Uri(_cleanURLFunc(newUri.AbsoluteUri));
+
                     if (!uris.Exists(u => u.AbsoluteUri == newUri.AbsoluteUri))
                         uris.Add(newUri);
                 }

Original file line number	Diff line number	Diff line change
`@@ -427,8 +427,8 @@ public void GetLinks_CleanUrlDelegateSet_ReturnsCleanLinks()`
`427`	`427`
`428`	`428`	`Assert.IsNotNull(result);`
`429`	`429`	`Assert.AreEqual(2, result.Count());`
`430`		`- Assert.AreEqual("http://a.com/xxx/x.html", result.ElementAt(0).AbsoluteUri);`
`431`		`- Assert.AreEqual("http://a.com/yyy/y.html", result.ElementAt(1).AbsoluteUri);`
	`430`	`+ Assert.AreEqual("http://x.com/xxx/x.html", result.ElementAt(0).AbsoluteUri);`
	`431`	`+ Assert.AreEqual("http://x.com/yyy/y.html", result.ElementAt(1).AbsoluteUri);`
`432`	`432`	`}`
`433`	`433`
`434`	`434`	`[Test] //https://github.com/sjdirect/abot/issues/15`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,6 @@ namespace Abot.Core`
`12`	`12`	`[Serializable]`
`13`	`13`	`public class HapHyperLinkParser : HyperLinkParser`
`14`	`14`	`{`
`15`		`- Func<string, string> _cleanURLFunc;`
`16`	`15`	`bool _isRespectAnchorRelNoFollowEnabled;`
`17`	`16`
`18`	`17`	`protected override string ParserType`
`@@ -36,10 +35,9 @@ public HapHyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled,`
`36`	`35`	`bool isRespectAnchorRelNoFollowEnabled,`
`37`	`36`	`Func<string, string> cleanURLFunc = null,`
`38`	`37`	`bool isRespectUrlNamedAnchorOrHashbangEnabled = false)`
`39`		`- :base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled)`
	`38`	`+ :base(isRespectMetaRobotsNoFollowEnabled, isRespectUrlNamedAnchorOrHashbangEnabled, cleanURLFunc)`
`40`	`39`	`{`
`41`	`40`	`_isRespectAnchorRelNoFollowEnabled = isRespectAnchorRelNoFollowEnabled;`
`42`		`- _cleanURLFunc = cleanURLFunc;`
`43`	`41`	`}`
`44`	`42`
`45`	`43`	`protected override IEnumerable<string> GetHrefValues(CrawledPage crawledPage)`
`@@ -94,7 +92,7 @@ protected virtual List<string> GetLinks(HtmlNodeCollection nodes)`
`94`	`92`	`if (HasRelNoFollow(node))`
`95`	`93`	`continue;`
`96`	`94`
`97`		`- hrefValue = _cleanURLFunc != null ? _cleanURLFunc(node.Attributes["href"].Value) : node.Attributes["href"].Value;`
	`95`	`+ hrefValue = node.Attributes["href"].Value;`
`98`	`96`	`if (!string.IsNullOrWhiteSpace(hrefValue))`
`99`	`97`	`{`
`100`	`98`	`hrefValue = DeEntitize(hrefValue);`
Original file line number	Diff line number	Diff line change
`@@ -23,18 +23,20 @@ public abstract class HyperLinkParser : IHyperLinkParser`
`23`	`23`	`{`
`24`	`24`	`protected ILog _logger = LogManager.GetLogger("AbotLogger");`
`25`	`25`	`protected bool _isRespectMetaRobotsNoFollowEnabled;`
`26`		`- protected bool _isRespectUrlNamedAnchorOrHashbangEnabled;`
	`26`	`+ protected bool _isRespectUrlNamedAnchorOrHashbangEnabled;`
	`27`	`+ protected Func<string, string> _cleanURLFunc;`
`27`	`28`
`28`	`29`	`public HyperLinkParser()`
`29`		`- :this(false, true)`
	`30`	`+ :this(false, true, null)`
`30`	`31`	`{`
`31`	`32`
`32`	`33`	`}`
`33`	`34`
`34`		`- public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectUrlNamedAnchorOrHashbangEnabled = false)`
	`35`	`+ public HyperLinkParser(bool isRespectMetaRobotsNoFollowEnabled, bool isRespectUrlNamedAnchorOrHashbangEnabled, Func<string, string> cleanURLFunc)`
`35`	`36`	`{`
`36`	`37`	`_isRespectMetaRobotsNoFollowEnabled = isRespectMetaRobotsNoFollowEnabled;`
`37`	`38`	`_isRespectUrlNamedAnchorOrHashbangEnabled = isRespectUrlNamedAnchorOrHashbangEnabled;`
	`39`	`+ _cleanURLFunc = cleanURLFunc;`
`38`	`40`	`}`
`39`	`41`
`40`	`42`	`/// <summary>`
`@@ -105,6 +107,9 @@ protected virtual List<Uri> GetUris(CrawledPage crawledPage, IEnumerable<string>`
`105`	`107`	`: hrefValue.Split('#')[0];`
`106`	`108`	`Uri newUri = new Uri(uriToUse, href);`
`107`	`109`
	`110`	`+ if (_cleanURLFunc != null)`
	`111`	`+ newUri = new Uri(_cleanURLFunc(newUri.AbsoluteUri));`
	`112`	`+`
`108`	`113`	`if (!uris.Exists(u => u.AbsoluteUri == newUri.AbsoluteUri))`
`109`	`114`	`uris.Add(newUri);`
`110`	`115`	`}`