bump of version nbr for this new working version, added a shortlink follower in utils.py

ianozsvald · ianozsvald · commit 19e2368e3e41 · 2013-03-28T14:52:31.000Z
diff --git a/README.rst b/README.rst
@@ -44,6 +44,14 @@ You can also ask for the span tags to be returned for each entity::
     [('https://github.com/ianozsvald/', (57, 87))]
 
 
+To use the shortlink follower:
+
+    >>> from ttp import utils
+    >>> # assume that result.urls == ['http://t.co/8o0z9BbEMu', u'http://bbc.in/16dClPF']
+    >>> print utils.follow_shortlinks(result.urls)  # pass in list of shortlink URLs
+    {'http://t.co/8o0z9BbEMu': [u'http://t.co/8o0z9BbEMu', u'http://bbc.in/16dClPF', u'http://www.bbc.co.uk/sport/0/21711199#TWEET650562'], u'http://bbc.in/16dClPF': [u'http://bbc.in/16dClPF', u'http://www.bbc.co.uk/sport/0/21711199#TWEET650562']}
+     >>> # note that bad shortlink URLs have a key to an empty list (lost/forgotten shortlink URLs don't generate any error)
+
 
 Installation
 ------------
@@ -61,6 +69,7 @@ Changelog
 ---------
 
  * 2013/2/11 1.0.0.2 released to PyPI
+ * 2013/4/? 1.0.1 new working version
 
 
 Tests
diff --git a/ttp/ttp.py b/ttp/ttp.py
@@ -23,7 +23,7 @@
 import re
 import urllib
 
-__version__ = "1.0.0.2"
+__version__ = "1.0.1.0"
 
 # Some of this code has been translated from the twitter-text-java library:
 # <http://github.com/mzsanford/twitter-text-java>
diff --git a/ttp/utils.py b/ttp/utils.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Unwind short-links e.g. bit.ly, t.co etc to their canonical links"""
+import requests
+
+
+def follow_shortlinks(shortlinks):
+    """Follow redirects in list of shortlinks, return dict of resulting URLs"""
+    links_followed = {}
+    for shortlink in shortlinks:
+        url = shortlink
+        request_result = requests.get(url)
+        redirect_history = request_result.history
+        # history might look like:
+        # (<Response [301]>, <Response [301]>)
+        # where each response object has a URL
+        all_urls = []
+        for redirect in redirect_history:
+            all_urls.append(redirect.url)
+        # append the final URL that we finish with
+        all_urls.append(request_result.url)
+        links_followed[shortlink] = all_urls
+    return links_followed
+
+
+if __name__ == "__main__":
+    shortlinks = ['http://t.co/8o0z9BbEMu', u'http://bbc.in/16dClPF']
+    print follow_shortlinks(shortlinks)
