Skip to content

Commit af0bd20

Browse files
bronsabronsa
committed
Merge branch 'TRDR-54'
2 parents 8e642fa + 130d845 commit af0bd20

File tree

3 files changed

+125
-30
lines changed

3 files changed

+125
-30
lines changed

‎src/main/clojure/clojure/tools/reader/reader_types.clj

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -102,23 +102,25 @@
102102
(when (instance? Closeable rdr)
103103
(.close ^Closeable rdr))))
104104

105-
(defn- normalize-newline [rdr ch]
106-
(if (identical? \return ch)
107-
(let [c (peek-char rdr)]
108-
(when (or (identical? \formfeed c)
109-
(identical? \newline c))
110-
(read-char rdr))
111-
\newline)
112-
ch))
113-
114105
(deftype IndexingPushbackReader
115106
[rdr ^:unsynchronized-mutable ^long line ^:unsynchronized-mutable ^long column
116107
^:unsynchronized-mutable line-start? ^:unsynchronized-mutable prev
117-
^:unsynchronized-mutable ^long prev-column file-name]
108+
^:unsynchronized-mutable ^long prev-column file-name
109+
^:unsynchronized-mutable normalize?]
118110
Reader
119111
(read-char [reader]
120112
(when-let [ch (read-char rdr)]
121-
(let [ch (normalize-newline rdr ch)]
113+
(let [ch (if normalize?
114+
(do (set! normalize? false)
115+
(if (or (identical? \newline ch)
116+
(identical? \formfeed ch))
117+
(read-char rdr)
118+
ch))
119+
ch)
120+
ch (if (identical? \return ch)
121+
(do (set! normalize? true)
122+
\newline)
123+
ch)]
122124
(set! prev line-start?)
123125
(set! line-start? (newline? ch))
124126
(when line-start?
@@ -138,7 +140,23 @@
138140
(set! column prev-column))
139141
(update! column dec))
140142
(set! line-start? prev)
141-
(unread rdr ch))
143+
;; This may look a bit convoluted, but it helps in the following
144+
;; scenario:
145+
;; + The underlying reader is about to return \return from the
146+
;; next read-char, and then \newline after that.
147+
;; + read-char gets \return, sets normalize? to true, returns
148+
;; \newline instead.
149+
;; + Caller calls unread on the \newline it just got. If we
150+
;; unread the \newline to the underlying reader, now it is ready
151+
;; to return two \newline chars in a row, which will throw off
152+
;; the tracked line numbers.
153+
(let [ch (if normalize?
154+
(do (set! normalize? false)
155+
(if (identical? \newline ch)
156+
\return
157+
ch))
158+
ch)]
159+
(unread rdr ch)))
142160

143161
IndexingReader
144162
(get-line-number [reader] (int line))
@@ -157,7 +175,7 @@
157175
(read-char [rdr]
158176
(let [c (.read ^java.io.PushbackReader rdr)]
159177
(when (>= c 0)
160-
(normalize-newline rdr (char c)))))
178+
(char c))))
161179

162180
(peek-char [rdr]
163181
(when-let [c (read-char rdr)]
@@ -245,11 +263,22 @@
245263
(deftype SourceLoggingPushbackReader
246264
[rdr ^:unsynchronized-mutable ^long line ^:unsynchronized-mutable ^long column
247265
^:unsynchronized-mutable line-start? ^:unsynchronized-mutable prev
248-
^:unsynchronized-mutable ^long prev-column file-name source-log-frames]
266+
^:unsynchronized-mutable ^long prev-column file-name source-log-frames
267+
^:unsynchronized-mutable normalize?]
249268
Reader
250269
(read-char [reader]
251270
(when-let [ch (read-char rdr)]
252-
(let [ch (normalize-newline rdr ch)]
271+
(let [ch (if normalize?
272+
(do (set! normalize? false)
273+
(if (or (identical? \newline ch)
274+
(identical? \formfeed ch))
275+
(read-char rdr)
276+
ch))
277+
ch)
278+
ch (if (identical? \return ch)
279+
(do (set! normalize? true)
280+
\newline)
281+
ch)]
253282
(set! prev line-start?)
254283
(set! line-start? (newline? ch))
255284
(when line-start?
@@ -347,7 +376,7 @@
347376
(indexing-push-back-reader s-or-rdr buf-len nil))
348377
([s-or-rdr buf-len file-name]
349378
(IndexingPushbackReader.
350-
(to-pbr s-or-rdr buf-len) 1 1 true nil 0 file-name)))
379+
(to-pbr s-or-rdr buf-len) 1 1 true nil 0 file-name false)))
351380

352381
(defn ^Closeable source-logging-push-back-reader
353382
"Creates a SourceLoggingPushbackReader from a given string or PushbackReader"
@@ -366,7 +395,8 @@
366395
file-name
367396
(doto (make-var)
368397
(alter-var-root (constantly {:buffer (StringBuilder.)
369-
:offset 0}))))))
398+
:offset 0})))
399+
false)))
370400

371401
(defn read-line
372402
"Reads a line from the reader or from *in* if no reader is specified"

‎src/test/clojure/clojure/tools/metadata_test.clj

Lines changed: 54 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
[clojure.string :as str]
77
[clojure.walk :as walk])
88
(:import java.nio.charset.Charset
9-
(java.io StringReader)
9+
(java.io StringReader BufferedReader)
1010
clojure.lang.LineNumberingPushbackReader))
1111

1212
(defn compare-forms-with-meta [expected-form actual-form]
@@ -28,9 +28,12 @@
2828

2929
(defn test-reader
3030
"Return a fresh byte array input stream reading off test-bytes"
31-
[]
31+
[test-contents]
3232
(StringReader. test-contents))
3333

34+
(defn replace-newlines [s replacement]
35+
(str/replace s "\n" replacement))
36+
3437
(def expected-haiku-ns
3538
(with-meta '(^{:line 1 :column 2 :end-line 1 :end-column 4 :file "haiku.clj"} ns
3639
^{:line 1 :column 5 :end-line 1 :end-column 31 :file "haiku.clj"} clojure.tools.reader.haiku)
@@ -56,15 +59,29 @@
5659
{:line 8 :column 5 :end-line 10 :end-column 32 :file "haiku.clj"}))
5760
{:line 3 :column 1 :end-line 10 :end-column 33 :file "haiku.clj"}))
5861

59-
(deftest read-metadata
60-
(let [reader (-> (test-reader)
61-
(LineNumberingPushbackReader.)
62-
(reader-types/indexing-push-back-reader 1 "haiku.clj"))
63-
first-form (read reader)
64-
second-form (read reader)]
62+
(defn multiple-reader-variants-from-string [s filename]
63+
[(-> (test-reader s)
64+
(LineNumberingPushbackReader.)
65+
(reader-types/indexing-push-back-reader 1 filename))
66+
(-> (test-reader s)
67+
(BufferedReader.)
68+
(reader-types/indexing-push-back-reader 1 filename))])
69+
70+
(defn read-metadata-helper [reader]
71+
(let [first-form (read reader)
72+
second-form (read reader)
73+
third-form (read reader false :eof)]
6574
(is (= {:line 1 :column 1 :end-line 1 :end-column 32 :file "haiku.clj"} (meta first-form)))
6675
(compare-forms-with-meta expected-haiku-ns first-form)
67-
(compare-forms-with-meta expected-haiku-defn second-form)))
76+
(compare-forms-with-meta expected-haiku-defn second-form)
77+
(is (= :eof third-form))))
78+
79+
(deftest read-metadata
80+
(doseq [s [test-contents
81+
(replace-newlines test-contents "\r")
82+
(replace-newlines test-contents "\r\n")]
83+
rdr (multiple-reader-variants-from-string s "haiku.clj")]
84+
(read-metadata-helper rdr)))
6885

6986
(def expected-haiku-ns-with-source
7087
(with-meta '(^{:line 1 :column 2 :end-line 1 :end-column 4 :source "ns" :file "haiku.clj"} ns
@@ -101,8 +118,8 @@
101118
^{:last last-five} [1 2 3])
102119
first-five middle-seven))" :file "haiku.clj"}))
103120

104-
(deftest read-metadata-with-source
105-
(let [reader (-> (test-reader)
121+
(defn read-metadata-with-source-helper [rdr]
122+
(let [reader (-> rdr
106123
(LineNumberingPushbackReader.)
107124
(reader-types/source-logging-push-back-reader 1 "haiku.clj"))
108125
first-form (read reader)
@@ -111,6 +128,12 @@
111128
(compare-forms-with-meta expected-haiku-ns-with-source first-form)
112129
(compare-forms-with-meta expected-haiku-defn-with-source second-form)))
113130

131+
(deftest read-metadata-with-source
132+
(doseq [s [test-contents
133+
(replace-newlines test-contents "\r")
134+
(replace-newlines test-contents "\r\n")]]
135+
(read-metadata-with-source-helper (test-reader s))))
136+
114137

115138
(def test2-contents
116139
(str/join "\n"
@@ -157,3 +180,23 @@
157180
(reader-types/indexing-push-back-reader 1 "vector.clj"))
158181
first-form (read reader)]
159182
(compare-forms-with-meta expected-vector first-form)))
183+
184+
(defn test-string [n linesep]
185+
(apply str (concat ["a "] (repeat n linesep) [" b"])))
186+
187+
(deftest many-consecutive-lineseps
188+
;; With older versions of tools.reader, consecutive-lineseps of
189+
;; 10,000, linesep "\r", and one of the variants of reader, would
190+
;; cause a StackOverflowError exception.
191+
(doseq [consecutive-lineseps [1 10 10000]
192+
linesep ["\n" "\r" "\r\n"]
193+
reader (multiple-reader-variants-from-string
194+
(test-string consecutive-lineseps linesep) "foo.clj")]
195+
(let [first-form (read reader)
196+
second-form (read reader)
197+
third-form (read reader false :eof)]
198+
(is (= {:line 1 :column 1 :end-line 1 :end-column 2 :file "foo.clj"} (meta first-form)))
199+
(is (= {:line (inc consecutive-lineseps) :column 2
200+
:end-line (inc consecutive-lineseps) :end-column 3 :file "foo.clj"}
201+
(meta second-form)))
202+
(is (= :eof third-form)))))

‎src/test/clojure/clojure/tools/reader_test.clj

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
(ns clojure.tools.reader-test
22
(:refer-clojure :exclude [read read-string *default-data-reader-fn* *data-readers*])
33
(:use [clojure.tools.reader :only [read read-string *default-data-reader-fn* *data-readers*]]
4-
[clojure.tools.reader.reader-types :only [string-push-back-reader]]
4+
[clojure.tools.reader.reader-types :only [string-push-back-reader
5+
indexing-push-back-reader]]
56
[clojure.test :only [deftest is are testing]]
67
[clojure.tools.reader.impl.utils :exclude [char]])
7-
(:import clojure.lang.BigInt))
8+
(:require [clojure.tools.reader.edn :as tre])
9+
(:import clojure.lang.BigInt
10+
(java.io StringReader BufferedReader)
11+
clojure.lang.LineNumberingPushbackReader))
812

913
(load "common_tests")
1014

@@ -177,3 +181,21 @@
177181
(is (= {::foo 1 :bar 2} (read-string "#::{:foo 1 :_/bar 2}")))
178182
(is (= {:a/foo 1 :bar 2} (read-string "#:a{:foo 1 :_/bar 2}")))
179183
(is (= {:clojure.core/foo 2} (read-string "#::c.c{:foo 2}")))))
184+
185+
(defn multiple-reader-variants-from-string [s filename]
186+
[(-> (StringReader. s)
187+
(LineNumberingPushbackReader.)
188+
(indexing-push-back-reader 1 filename))
189+
(-> (StringReader. s)
190+
(BufferedReader.)
191+
(indexing-push-back-reader 1 filename))])
192+
193+
(defn first-reads-from-multiple-readers [s]
194+
(for [rdr (multiple-reader-variants-from-string s "file.edn")]
195+
(tre/read rdr)))
196+
197+
(deftest trdr-54
198+
(let [read-vals (mapcat first-reads-from-multiple-readers
199+
["[a\rb]" "[a\r b]" "[a \rb]"])]
200+
(doseq [pairs (partition 2 1 read-vals)]
201+
(is (= (first pairs) (second pairs))))))

0 commit comments

Comments
 (0)