* Changed how we get the redirect URI following PR comment

mikegron · mikegron · commit 369d6fb095e8 · 2015-09-17T11:58:30.000-04:00
diff --git a/Abot/Crawler/WebCrawler.cs b/Abot/Crawler/WebCrawler.cs
@@ -724,7 +724,7 @@ protected virtual void ProcessRedirect(CrawledPage crawledPage)
                 
             try
             {
-                var uri = GetRedirectUri(crawledPage);
+                var uri = ExtractRedirectUri(crawledPage);
 
                 PageToCrawl page = new PageToCrawl(uri);
                 page.ParentUri = crawledPage.ParentUri;
@@ -1042,7 +1042,7 @@ protected virtual void ValidateRootUriForRedirection(CrawledPage crawledRootPage
             }
 
             if (IsRedirect(crawledRootPage)) {
-                _crawlContext.RootUri = GetRedirectUri(crawledRootPage);
+                _crawlContext.RootUri = ExtractRedirectUri(crawledRootPage);
                 _logger.InfoFormat("The root URI [{0}] was redirected to [{1}]. Pages from domains [{2}] and [{3}] will be considered internal.",
                     _crawlContext.OriginalRootUri,
                     _crawlContext.RootUri,
@@ -1058,7 +1058,7 @@ protected virtual void ValidateRootUriForRedirection(CrawledPage crawledRootPage
         /// If HTTP auto redirections is disabled, this value is stored in the 'Location' header of the response.
         /// If auto redirections is enabled, this value is stored in the response's ResponseUri property.
         /// </remarks>
-        protected virtual Uri GetRedirectUri(CrawledPage crawledPage)
+        protected virtual Uri ExtractRedirectUri(CrawledPage crawledPage)
         {
             Uri locationUri;
             if (_crawlContext.CrawlConfiguration.IsHttpRequestAutoRedirectsEnabled) {
@@ -1067,12 +1067,12 @@ protected virtual Uri GetRedirectUri(CrawledPage crawledPage)
             } else {
                 // For manual redirects, we need to look for the location header.
                 var location = crawledPage.HttpWebResponse.Headers["Location"];
-                
+
+                // Check if the location is absolute. If not, create an absolute uri.
                 if (!Uri.TryCreate(location, UriKind.Absolute, out locationUri))
                 {
-                    var site = crawledPage.Uri.Scheme + "://" + crawledPage.Uri.Host;
-                    location = site + location;
-                    locationUri = new Uri(location);
+                    Uri baseUri = new Uri(crawledPage.Uri.GetLeftPart(UriPartial.Authority));
+                    locationUri = new Uri(baseUri, location);
                 }
             }
             return locationUri;
