Skip to content

Commit 4c6cf2b

Browse files
committed
wordcount
1 parent 837da3b commit 4c6cf2b

File tree

22 files changed

+561
-244
lines changed

22 files changed

+561
-244
lines changed

app/views/cljs_om.scala.html

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ <h5>word frequency</h5>
139139
<script src="/assets/javascripts/vendor/underscore.min.js"></script>
140140
<script src="/assets/react-js/vendor/react.js"></script>
141141
<script src="/assets/react-js/vendor/regression.js"></script>
142-
142+
<script src="/assets/cljs-js/barchart.js"></script>
143+
<script src="/assets/cljs-js/wordcloud.js"></script>
143144

144145
<script src="/assets/cljs/out/goog/base.js" type="text/javascript"></script>
145146
<script src="/assets/cljs/cljs_om.js" type="text/javascript"></script>

cljs-om/externs.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
var moment = {};
22
moment.fromNow = function(bool) {};
3+
4+
var BirdWatch = {};
5+
BirdWatch.WordCloud = function (w, h, maxEntries, addSearch, elem) {};

cljs-om/src/cljs_om/core.cljs

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,29 @@
22
(:require [om.core :as om :include-macros true]
33
[om.dom :as dom :include-macros true]
44
[cljs-om.util :as util]
5-
[cljs-om.ui :as ui]))
5+
[cljs-om.ui :as ui]
6+
[cljs-om.wordcount :as wc]))
67

78
(enable-console-print!)
89

9-
(defn sort-by [key]
10+
(defn sort-by [key-a key-b]
1011
"sorting function, initially comparing specified key and, if equal, favors higher ID"
1112
(fn [x y]
12-
(if (not (= (key x) (key y)))
13-
(> (key x) (key y))
14-
(> (:id x) (:id y)))))
13+
(if (not (= (key-a x) (key-a y)))
14+
(> (key-a x) (key-a y))
15+
(> (key-b x) (key-b y)))))
1516

16-
(def initial-state {:count 0
17-
:tweets-map {}
18-
:rt-since-startup {}
19-
:by-followers (sorted-set-by (sort-by :followers_count))
20-
:by-retweets (sorted-set-by (sort-by :retweet_count))
21-
:by-rt-since-startup (sorted-set-by (sort-by :count))
22-
:by-favorites (sorted-set-by (sort-by :favorite_count))
23-
:by-id (sorted-set-by >)
24-
:n 10
17+
(def initial-state {:count 0 :n 10
18+
:tweets-map {} :rt-since-startup {}
19+
:search "*" :stream nil
2520
:sorted :by-followers
26-
:search "*"
27-
:stream nil})
21+
:by-followers (sorted-set-by (sort-by :followers_count :id))
22+
:by-retweets (sorted-set-by (sort-by :retweet_count :id))
23+
:by-rt-since-startup (sorted-set-by (sort-by :count :id))
24+
:by-favorites (sorted-set-by (sort-by :favorite_count :id))
25+
:by-id (sorted-set-by >)
26+
:words {}
27+
:words-sorted-by-count (sorted-set-by (sort-by :value :key))})
2828

2929
(def app-state (atom initial-state))
3030

@@ -61,15 +61,22 @@
6161
(mod-sort-set :by-retweets conj :retweet_count (:retweet_count rt) rt)
6262
(mod-sort-set :by-favorites conj :favorite_count (:favorite_count rt) rt))))
6363

64+
(def cloud-elem (. js/document (getElementById "wordCloud")))
65+
(def cloud-h (aget cloud-elem "offsetWidth"))
66+
(def word-cloud (.WordCloud js/BirdWatch cloud-h (* cloud-h 0.7) 250 (fn [e]) "#wordCloud"))
67+
6468
(defn add-tweet [tweet]
6569
"increment counter, add tweet to tweets map and to sorted sets by id and by followers"
6670
(swap! app-state assoc :count (inc (:count @app-state)))
6771
(add-to-tweets-map tweet)
6872
(add-rt-status tweet)
73+
(wc/process-tweet app-state (:text tweet))
6974
(swap! app-state assoc :by-followers (conj (:by-followers @app-state)
7075
{:followers_count (:followers_count (:user tweet))
7176
:id (:id_str tweet)}))
72-
(swap! app-state assoc :by-id (conj (:by-id @app-state) (:id_str tweet))) )
77+
(swap! app-state assoc :by-id (conj (:by-id @app-state) (:id_str tweet)))
78+
(. word-cloud (redraw (clj->js (take 250 (:words-sorted-by-count @app-state)))))
79+
(.updateBarchart js/BirdWatch (clj->js (take 25 (:words-sorted-by-count @app-state)))))
7380

7481
(defn receive-sse [e]
7582
"callback, called for each item (tweet) received by SSE stream"

cljs-om/src/cljs_om/ui.cljs

Lines changed: 14 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -12,33 +12,22 @@
1212
(render [this]
1313
(dom/span nil (:count app)))))
1414

15-
(defn tweets-by-followers [app n]
16-
"find top n tweets by followers of the author in descending order"
17-
(vec (map (fn [m] ((keyword (:id m))(:tweets-map app))) (take n (:by-followers app)))))
18-
19-
(defn tweets-by-retweets [app n]
20-
"find top n tweets by retweets in descending order"
21-
(vec (map (fn [m] ((keyword (:id m)) (:tweets-map app))) (take n (:by-retweets app)))))
22-
23-
(defn tweets-by-rt-since-startup [app n]
24-
"find top n tweets by retweets in descending order"
25-
(vec (map (fn [m] ((keyword (:id m)) (:tweets-map app))) (take n (:by-rt-since-startup app)))))
26-
27-
(defn tweets-by-favorites [app n]
28-
"find top n tweets by retweets in descending order"
29-
(vec (map (fn [m] ((keyword (:id m))(:tweets-map app))) (take n (:by-favorites app)))))
15+
(defn tweets-by-order [order]
16+
"find top n tweets by specified order"
17+
(fn [app n]
18+
(vec (map (fn [m] ((keyword (:id m))(:tweets-map app))) (take n (order app))))))
3019

3120
(defn tweets-by-id [app n]
3221
"find top n tweets sorted by ID in descending order"
3322
(vec (map (fn [m] ((keyword m)(:tweets-map app))) (take n (:by-id app)))))
3423

3524
(def find-tweets {:by-id tweets-by-id
36-
:by-followers tweets-by-followers
37-
:by-retweets tweets-by-retweets
38-
:by-favorites tweets-by-favorites
39-
:by-rt-since-startup tweets-by-rt-since-startup})
25+
:by-followers (tweets-by-order :by-followers)
26+
:by-retweets (tweets-by-order :by-retweets)
27+
:by-favorites (tweets-by-order :by-favorites)
28+
:by-rt-since-startup (tweets-by-order :by-rt-since-startup)})
4029

41-
(defn sort-button [app key]
30+
(defn sort-button-js [app key]
4231
#js {:onClick (fn [e] (om/update! app [:sorted] key))
4332
:className (str "btn " (if (= key (:sorted app)) "btn-primary"))})
4433

@@ -49,11 +38,11 @@
4938
(render [this]
5039
(dom/div #js {:className "btn-group"}
5140
(dom/button #js {:className "btn"} "Sort by")
52-
(dom/button (sort-button app :by-id) "latest")
53-
(dom/button (sort-button app :by-followers) "followers")
54-
(dom/button (sort-button app :by-retweets) "retweets")
55-
(dom/button (sort-button app :by-rt-since-startup) "retweets2")
56-
(dom/button (sort-button app :by-favorites) "favorites")))))
41+
(dom/button (sort-button-js app :by-id) "latest")
42+
(dom/button (sort-button-js app :by-followers) "followers")
43+
(dom/button (sort-button-js app :by-retweets) "retweets")
44+
(dom/button (sort-button-js app :by-rt-since-startup) "retweets2")
45+
(dom/button (sort-button-js app :by-favorites) "favorites")))))
5746

5847
(defn handle-change [e owner {:keys [text]}]
5948
(om/set-state! owner :text (.. e -target -value)))

cljs-om/src/cljs_om/wordcount.cljs

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,39 @@
1-
(ns cljs-om.wordcount)
1+
(ns cljs-om.wordcount
2+
(:require [clojure.string :as s]))
23

3-
(def words (atom {}))
4+
(def stop-words #{"use" "good" "want" "amp" "just" "now" "like" "til" "new" "get" "one" "i" "me" "my" "myself" "we"
5+
"us" "our" "ours" "ourselves" "you" "your" "yours" "yourself" "yourselves" "he" "him" "his" "himself"
6+
"she" "her" "hers" "herself" "it" "its" "itself" "they" "them" "their" "theirs" "themselves" "what"
7+
"which" "who" "whom" "whose" "this" "that" "these" "those" "am" "is" "are" "was" "were" "be" "been"
8+
"being" "have" "has" "had" "having" "do" "does" "did" "doing" "will" "would" "should" "can" "could"
9+
"ought" "i'm" "you're" "he's" "she's" "it's" "we're" "they're" "i've" "you've" "we've" "they've" "i'd"
10+
"you'd" "he'd" "she'd" "we'd" "they'd" "i'll" "you'll" "he'll" "she'll" "we'll" "they'll" "isn't"
11+
"aren't" "wasn't" "weren't" "hasn't" "haven't" "hadn't" "doesn't" "don't" "didn't" "won't" "wouldn't"
12+
"shan't" "shouldn't" "can't" "cannot" "couldn't" "mustn't" "let's" "that's" "who's" "what's" "here's"
13+
"there's" "when's" "where's" "why's" "how's" "a" "an" "the" "and" "but" "if" "or" "because" "as"
14+
"until" "while" "of" "at" "by" "for" "with" "about" "against" "between" "into" "through" "during"
15+
"before" "after" "above" "below" "to" "from" "up" "upon" "down" "in" "out" "on" "off" "over" "under"
16+
"again" "further" "then" "once" "here" "there" "when" "where" "why" "how" "all" "any" "both" "each"
17+
"few" "more" "most" "other" "some" "such" "no" "nor" "not" "only" "own" "same" "so" "than" "too"
18+
"very" "say" "says" "said" "shall" "via" "htt…" "don" "let" "gonna" "rt" "&amp" "http"})
19+
20+
(defn add-word [app word]
21+
(let [prev-count (get (:words @app) word)]
22+
(swap! app assoc :words-sorted-by-count (disj (:words-sorted-by-count @app) {:key word :value prev-count}))
23+
(swap! app assoc-in [:words word] (inc (get (:words @app) word)))
24+
(swap! app assoc :words-sorted-by-count (conj (:words-sorted-by-count @app) {:key word :value (+ prev-count 1)}))
25+
)
26+
word)
27+
28+
(defn process-tweet [app text]
29+
"process tweet: split, filter, lower case, replace punctuation, add word"
30+
(doall ;; initially lazy, needs realization
31+
(->> (s/split text #"[\s\u3031-\u3035\u0027\u309b\u309c\u30a0\u30fc\uff70]+")
32+
(filter #(not (re-find #"(@|https?:)" %)) ,)
33+
(filter #(> (count %) 3) ,)
34+
(filter #(< (count %) 25) ,)
35+
(map s/lower-case ,)
36+
(map #(s/replace % #"[;:,/‘’…~\-!?#<>()\"@.]+" "" ) ,)
37+
(filter (fn [item] (not (contains? stop-words item))) ,)
38+
(map #(add-word app %) ,))))
439

5-
(defn split-tweet [tweet])

public/cljs-js/barchart.js

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/** @jsx React.DOM */
2+
3+
var BirdWatch = BirdWatch || {};
4+
5+
(function () {
6+
7+
/** function preparing data for use by regression.js */
8+
function regressionData(hist) { return hist.map(function(el, idx, arr) { return [idx, -el]; }); }
9+
10+
/** Arrow component for use in BarChart */
11+
var Arrow = React.createClass({displayName: 'Arrow',
12+
render: function () {
13+
var y = parseInt(this.props.y);
14+
var arr = "-600,100 200,100 -200,500 100,500 600,0 100,-500 -200,-500 200,-100 -600,-100 ";
15+
var arrColor = "#428bca";
16+
if (this.props.dir === "UP") {
17+
arrColor = "#45cc40";
18+
arr = "100,600 100,-200 500,200 500,-100 0,-600 -500,-100 -500,200 -100,-200 -100,600";
19+
}
20+
if (this.props.dir === "RIGHT-UP") {
21+
arrColor = "#45cc40";
22+
arr ="400,-400 -200,-400 -350,-250 125,-250 -400,275 -275,400 250,-125 250,350 400,200";
23+
}
24+
if (this.props.dir === "DOWN") {
25+
arrColor = "#dc322f";
26+
arr = "100,-600 100,200 500,-200 500,100 0,600 -500,100 -500,-200 -100,200 -100,-600";
27+
}
28+
if (this.props.dir === "RIGHT-DOWN") {
29+
arrColor = "#dc322f";
30+
arr = "400,400 -200,400 -350,250 125,250 -400,-275 -275,-400 250,125 250,-350 400,-200";
31+
}
32+
var arrowTrans = "translate(" + this.props.x + ", "+ (y + 7) + ") scale(0.01) ";
33+
34+
return ( React.DOM.polygon( {transform:arrowTrans, stroke:"none", fill:arrColor, points:arr}) ); }
35+
});
36+
37+
function now () { return new Date().getTime(); }
38+
39+
/** single Bar component for assembling BarChart */
40+
var Bar = React.createClass({displayName: 'Bar',
41+
getInitialState: function() {
42+
return {ratioHist: [], posHist: [], lastUpdate: now(), posArrDir: "RIGHT", ratioArrDir: "RIGHT-UP"}
43+
},
44+
/** this function gets called right before rendering. component state can be modified here, not in render */
45+
componentWillReceiveProps: function(props) {
46+
this.setState({ratioHist: _.last(this.state.ratioHist.concat(props.val / props.count), this.props.ratioChangeTweets)});
47+
this.setState({posHist: _.last(this.state.posHist.concat(props.idx+1), 2)});
48+
49+
// slope of the fitted position change function
50+
var posSlope = regression('linear', regressionData(this.state.posHist)).equation[0];
51+
if (posSlope === 0 && now() - this.state.lastUpdate > this.props.posChangeDur) { this.setState({posArrDir: "RIGHT"}); }
52+
if (posSlope > 0) { this.setState({posArrDir: "UP", lastUpdate: now()}); }
53+
if (posSlope < 0) { this.setState({posArrDir: "DOWN", lastUpdate: now()}); }
54+
55+
// slope of the ratio (value / total value) change
56+
var ratioSlope = regression('linear', regressionData(this.state.ratioHist)).equation[0];
57+
this.setState({ratioArrDir: (ratioSlope > 0) ? "RIGHT-UP" : "RIGHT-DOWN"});
58+
},
59+
/** adding terms to the search bar when clicking on any of the bars */
60+
clickHandler: function(e) { BirdWatch.addSearchTerm(this.props.key); },
61+
render: function () {
62+
var y = parseInt(this.props.y);
63+
var t = this.props.t;
64+
var w = parseInt(this.props.w);
65+
var val = this.props.val;
66+
67+
var textX = w+145;
68+
var style = {fontWeight: 500, fill: "#DDD", textAnchor: "end"};
69+
if (w < 50) { style.fill="#999"; textX+=26; style.textAnchor="start"; style.fontWeight=400}
70+
71+
return React.DOM.g( {onClick:this.clickHandler},
72+
React.DOM.text( {y:y+12, x:"137", stroke:"none", fill:"black", dy:".35em", textAnchor:"end"}, t),
73+
Arrow( {dir:this.state.posArrDir, y:y, x:146} ),
74+
Arrow( {dir:this.state.ratioArrDir, y:y, x:160} ),
75+
React.DOM.rect( {y:y, x:"168", height:"15", width:w, stroke:"white", fill:"#428bca"}),
76+
React.DOM.text( {y:y+12, x:textX, stroke:"none", style:style, dy:".35em"} , val)
77+
)
78+
}
79+
});
80+
81+
/** BarChart component, renders all bar items.
82+
* Also renders interactive legend where the trend indicator durations can be configured */
83+
var BarChart = React.createClass({displayName: 'BarChart',
84+
render: function() {
85+
var h = this.props.words.length * 15;
86+
var bars = this.props.words.map(function (bar, i, arr) {
87+
if (!bar) return "";
88+
var y = i * 15;
89+
var w = bar.value / arr[0].value * (barChartElem.width() - 190);
90+
return Bar( {t:bar.key, y:y, w:w, key:bar.key, idx:i, val:bar.value,
91+
posChangeDur:this.refs.posChangeDur.getDOMNode().value,
92+
ratioChangeTweets:this.refs.ratioChangeTweets.getDOMNode().value} );
93+
}.bind(this));
94+
return React.DOM.div(null,
95+
React.DOM.svg( {width:"750", height:h},
96+
React.DOM.g(null,
97+
bars,
98+
React.DOM.line( {transform:"translate(168, 0)", y:"0", y2:h, stroke:"#000000"})
99+
)
100+
),
101+
React.DOM.p( {className:"legend"}, React.DOM.strong(null, "1st trend indicator:"), " position changes in last  ",
102+
React.DOM.select( {defaultValue:"300000", ref:"posChangeDur"},
103+
React.DOM.option( {value:"10000"}, "10 seconds"),
104+
React.DOM.option( {value:"30000"}, "30 seconds"),
105+
React.DOM.option( {value:"60000"}, "minute"),
106+
React.DOM.option( {value:"300000"}, "5 minutes"),
107+
React.DOM.option( {value:"600000"}, "10 minutes")
108+
)
109+
),
110+
React.DOM.p( {className:"legend"}, React.DOM.strong(null, "2nd trend indicator:"), " ratio change termCount / totalTermsCounted over last  ",
111+
React.DOM.select( {defaultValue:"100", ref:"ratioChangeTweets"},
112+
React.DOM.option( {value:"10"}, "10 tweets"),
113+
React.DOM.option( {value:"100"}, "100 tweets"),
114+
React.DOM.option( {value:"500"}, "500 tweets"),
115+
React.DOM.option( {value:"1000"}, "1000 tweets")
116+
)
117+
)
118+
)
119+
}
120+
});
121+
122+
var barChartElem = $("#react-bar-chart");
123+
var barChart = React.renderComponent(BarChart( {numPages:1, words:[]}), document.getElementById('react-bar-chart'));
124+
125+
BirdWatch.updateBarchart = function (words) { barChart.setProps({words: _.take(words, 25) }); };
126+
})();

0 commit comments

Comments
 (0)