Skip to content

Commit 965675b

Browse files
matthiasnmatthiasn
committed
added feature: sort order by reach
1 parent ce5c8f3 commit 965675b

File tree

8 files changed

+41
-42
lines changed

8 files changed

+41
-42
lines changed

‎Clojure-Websockets/project.clj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
:source-paths ["src/cljs"]
3535
:compiler {:output-to "resources/public/js/build/birdwatch.js"
3636
:output-dir "resources/public/js/build/out"
37-
:optimizations :simple
37+
:optimizations :none
3838
:source-map "resources/public/js/build/birdwatch.js"}}
3939
{:id "release"
4040
:source-paths ["src/cljs"]

‎Clojure-Websockets/resources/public/index.html

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ <h3>Usage</h3>
5656
<li><strong>followers</strong>: by the number of followers of the tweet's author</li>
5757
<li><strong>retweets</strong>: by the total retweets of a tweet</li>
5858
<li><strong>retweets2</strong>: by the number of retweets within the loaded tweets</li>
59+
<li><strong>reach</strong>: number of users that tweet was (re-)tweeted to within loaded tweets</li>
5960
<li><strong>favorites</strong>: by total number of times a tweet has been favorited</li>
6061
</ul>
6162
<p>

‎Clojure-Websockets/src/clj/birdwatch/persistence.clj

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -58,37 +58,29 @@
5858
"get vector with :_source of each ElasticSearch result"
5959
(map strip-source coll))
6060

61-
(defn native-query [params]
61+
(defn native-query [{:keys [query n from]}]
6262
"run a query on previous matching tweets"
63-
(let [conn (esn/connect [["127.0.0.1" 9300]] {"cluster.name" "elasticsearch_mn"})
64-
q (:query params)
65-
res (esnd/search conn (:es-index conf) "tweet" :query q :size (:n params) :from (:from params) :sort {:id :desc})
66-
n (esnrsp/total-hits res)
67-
hits (esnrsp/hits-from res)
68-
res (get-source hits)]
69-
(log/info "Total hits:" n "Retrieved:" (count hits))
70-
res))
63+
(let [native-conn (esn/connect [["127.0.0.1" 9300]] {"cluster.name" "elasticsearch_mn"})
64+
search (esnd/search native-conn (:es-index conf) "tweet" :query query :size n :from from :sort {:id :desc})
65+
hits (esnrsp/hits-from search)]
66+
(log/info "Total hits:" (esnrsp/total-hits search) "Retrieved:" (count hits))
67+
(get-source hits)))
7168

72-
(defn query [params]
69+
(defn query [{:keys [query n from]}]
7370
"run a query on previous matching tweets"
74-
(let [conn (esr/connect (:es-address conf))
75-
q (:query params)
76-
res (esd/search conn (:es-index conf) "tweet" :query q :size (:n params) :from (:from params) :sort {:id :desc})
77-
n (esrsp/total-hits res)
78-
hits (esrsp/hits-from res)
71+
(let [search (esd/search conn (:es-index conf) "tweet" :query query :size n :from from :sort {:id :desc})
72+
hits (esrsp/hits-from search)
7973
res (get-source hits)]
80-
;(log/info "Total hits:" n "Retrieved:" (count hits))
74+
(log/info "Total hits:" (esrsp/total-hits search) "Retrieved:" (count hits))
8175
;(log/info "top retweets in chunk" (pp/pprint (take 10 (into (priority-map-by >) (d/retweets res {})))))
8276
res))
8377

84-
(defn start-percolator [params]
78+
(defn start-percolator [{:keys [query uid]}]
8579
"register percolation search with ID based on hash of the query"
86-
(let [q (:query params)
87-
sha (sha1 (str q))
88-
uid (:uid params)]
80+
(let [sha (sha1 (str query))]
8981
(swap! a/subscriptions assoc uid sha)
90-
(perc/register-query conn "percolator" sha :query q)
91-
(log/info "Percolation registered for query" q "with SHA1" sha)))
82+
(perc/register-query conn "percolator" sha :query query)
83+
(log/info "Percolation registered for query" query "with SHA1" sha)))
9284

9385
;; loop for persisting tweets
9486
(go

‎Clojure-Websockets/src/cljs/birdwatch/state.cljs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
:by-retweets (priority-map-by >)
2525
:by-favorites (priority-map-by >)
2626
:by-rt-since-startup (priority-map-by >)
27+
:by-reach (priority-map-by >)
2728
:by-id (priority-map-by >)
2829
:words-sorted-by-count (priority-map-by >)})
2930

‎Clojure-Websockets/src/cljs/birdwatch/tweets.cljs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,19 @@
3838
(swap-when-larger app :by-retweets rt-id rt-count)
3939
(swap-when-larger app :by-favorites rt-id (:favorite_count rt))
4040
(util/swap-pmap app :by-rt-since-startup rt-id (inc (get (:by-rt-since-startup state) rt-id 0)))
41+
(util/swap-pmap app :by-reach rt-id (+ (get (:by-reach state) rt-id 0) (:followers_count (:user tweet))))
4142
(when (> rt-count (:retweet_count (rt-id (:tweets-map state)))) (add-to-tweets-map app :tweets-map rt)))))
4243

4344
(defn add-tweet [tweet app]
4445
"increment counter, add tweet to tweets map and to sorted sets by id and by followers"
45-
(let [state @app]
46+
(let [state @app
47+
id-str (:id_str tweet)
48+
id-key (keyword id-str)]
4649
(swap! app assoc :count (inc (:count state)))
4750
(add-to-tweets-map app :tweets-map tweet)
48-
(util/swap-pmap app :by-followers (keyword (:id_str tweet)) (:followers_count (:user tweet)))
49-
(util/swap-pmap app :by-id (keyword (:id_str tweet)) (:id_str tweet))
51+
(util/swap-pmap app :by-followers (keyword id-str) (:followers_count (:user tweet)))
52+
(util/swap-pmap app :by-id (keyword id-str) id-str)
53+
(util/swap-pmap app :by-reach id-key (+ (get (:by-reach state) id-key 0) (:followers_count (:user tweet))))
5054
(add-rt-status app tweet)
5155
(wc/process-tweet app (:text tweet))))
5256

‎Clojure-Websockets/src/cljs/birdwatch/ui-tweets.cljs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,10 @@
3838
(dom/div #js {:className "pull-left timeInterval"}
3939
(str (util/number-format (:followers_count user)) " followers"))
4040
(dom/div #js {:className "pull-right timeInterval"}
41-
(str (util/rt-count-since-startup tweet)
42-
(util/rt-count tweet)
43-
(util/fav-count tweet))))
41+
(str (util/rt-count tweet)
42+
(util/fav-count tweet))
43+
(dom/br nil)
44+
(util/rt-count-since-startup tweet)))
4445
(when (> (count media) 0)
4546
(dom/div #js {:className "tweet-image"}
4647
(dom/a #js {:href (:url (get media 0)) :target "_blank"}

‎Clojure-Websockets/src/cljs/birdwatch/ui.cljs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
(dom/button (sort-button-js app :by-followers) "followers")
4848
(dom/button (sort-button-js app :by-retweets) "retweets")
4949
(dom/button (sort-button-js app :by-rt-since-startup) "retweets2")
50+
(dom/button (sort-button-js app :by-reach) "reach")
5051
(dom/button (sort-button-js app :by-favorites) "favorites")))))
5152

5253
(defn handle-search-change [e app]

‎Clojure-Websockets/src/cljs/birdwatch/util.cljs

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,14 @@
4242

4343
(defn format-tweet [tweet]
4444
"format tweet text for display"
45-
(assoc tweet :html-text
46-
(-> (:text tweet)
47-
(reducer , (:urls (:entities tweet)) url-replacer)
48-
(reducer , (:media (:entities tweet)) url-replacer)
49-
(reducer , (:user_mentions (:entities tweet)) mentions-replacer)
50-
(reducer , (:hashtags (:entities tweet)) hashtags-replacer)
51-
(s/replace , "RT " "<strong>RT </strong>"))))
45+
(let [{:keys [urls media user_mentions hashtags]} (:entities tweet)]
46+
(assoc tweet :html-text
47+
(-> (:text tweet)
48+
(reducer , urls url-replacer)
49+
(reducer , media url-replacer)
50+
(reducer , user_mentions mentions-replacer)
51+
(reducer , hashtags hashtags-replacer)
52+
(s/replace , "RT " "<strong>RT </strong>")))))
5253

5354
(defn entity-count [tweet sym s]
5455
"gets count of specified entity from either tweet, or, when exists, original (retweeted) tweet"
@@ -62,23 +63,21 @@
6263
(defn rt-count-since-startup [tweet]
6364
"gets RT count since startup for tweet, if exists returns formatted string"
6465
(let [t (if (contains? tweet :retweeted_status) (:retweeted_status tweet) tweet)
65-
count ((keyword (:id_str t)) (:by-rt-since-startup @state/app))]
66-
(if (> count 0) (str (number-format count) " RTs analyzed | ") "")))
66+
cnt ((keyword (:id_str t)) (:by-rt-since-startup @state/app))
67+
reach ((keyword (:id_str t)) (:by-reach @state/app))]
68+
(if (> cnt 0) (str "analyzed: " (number-format cnt) " retweets, reach " (number-format reach)))))
6769

6870
(defn swap-pmap [app priority-map id n]
6971
"swaps item in priority-map"
7072
(swap! app assoc priority-map (assoc (priority-map @app) id n)))
7173

72-
7374
(defn tweets-by-order [tweets-map order]
7475
"find top n tweets by specified order"
7576
(fn [app n skip]
76-
(let [tw-map tweets-map ord order]
77-
(vec (map (fn [m] ((keyword (first m))(tw-map app))) (take n (drop (* n skip) (ord app))))))))
77+
(vec (map (fn [m] ((keyword (first m))(tweets-map app))) (take n (drop (* n skip) (order app)))))))
7878

7979
(defn tweets-by-order2 [order app n skip]
8080
"find top n tweets by specified order"
81-
;(print order)
8281
(vec
8382
(filter identity
8483
(map

0 commit comments

Comments
 (0)