Work-around tools reader newline behaviour

I've reported what, seems to me, curious behaviour in clojure tools
reader. See https://clojure.atlassian.net/browse/TRDR-65

Instead of waiting for a resolution from the tools reader project,
I have taken a crack at a work-around.

In short:
1. For Clojure: insert a custom newline normalizing reader into the
   reader chain.
2. For ClojueScript: compensate for the fact tools reader peek-char
   returns \r instead of \n for \r\n.

As part of this work, now only calling clojure.tools.reader APIs from
rewrite-clj.reader namespace. Localizing access to clojure.tools.reader
through a single point ensures that any fixes/customizations we have
atop the clojure.tools.reader get used. Includes:
- moving customized version of read-keyword fn from
  rewrite-clj.keyword.parser to rewrite-clj.reader namespace.
- rewrite-clj.parser.util fns were mostly duplicated in
  rewrite-clj.reader. Turf rewrite-clj.parser and have parser namespaces
  not be shy to access rewrite-clj.reader namespace instead.
  Logged as potentially breaking in change log.

Closes #93

Author: lread

CHANGELOG.adoc

@@ -58,6 +58,7 @@ Rewrite-cljs users migrating to rewrite-clj v1 are now at, and will remain at, f
 * Many updates to docs and docstrings
 
 ==== Fixes
+* end of line variants in source now normalized consistently to `\newline` https://github.com/clj-commons/rewrite-clj/issues/93[#93]
 * postwalk on larger source file no longer throws StackOverflow https://github.com/clj-commons/rewrite-clj/issues/69[#69]
 * postwalk now walks in post order https://github.com/clj-commons/rewrite-clj/issues/123[#123]
 * we now preserve newline at end of file https://github.com/clj-commons/rewrite-clj/issues/121[#121]
@@ -129,6 +130,8 @@ Rewrite-cljs users migrating to rewrite-clj v1 are now at, and will remain at, f
 ** Switched to `clojure.tools.reader.edn`
 ** Some rewrite-cljs optimizations were dropped in favor of a single code base
 ** Deleted unused `rewrite-clj.node.indent` https://github.com/clj-commons/rewrite-clj/issues/116[#116]
+** Deleted redundent `rewrite-clj.parser.util` as part of https://github.com/clj-commons/rewrite-clj/issues/93[#93].
+If you were using this internal namespace you can opt to switch to, the also internal, `rewrite-clj.reader` namespace.
 
 == rewrite-clj v0
 

src/rewrite_clj/parser/keyword.cljc

@@ -1,44 +1,18 @@
 (ns ^:no-doc rewrite-clj.parser.keyword
-  (:require [clojure.tools.reader.edn :as edn]
-            ;; internal tools reader namespaces to support read-keyword override work
-            [clojure.tools.reader.impl.commons :as reader-impl-commons]
-            [clojure.tools.reader.impl.errors :as reader-impl-errors]
-            [clojure.tools.reader.impl.utils :as reader-impl-utils]
-            [clojure.tools.reader.reader-types :as r]
-            [rewrite-clj.node.keyword :as nkeyword]
-            [rewrite-clj.parser.utils :as u] ))
+  (:require [rewrite-clj.node.keyword :as nkeyword]
+            [rewrite-clj.reader :as reader]))
 
 #?(:clj (set! *warn-on-reflection* true))
 
-(defn- read-keyword
-  "This customized version of clojure.tools.reader.edn's read-keyword allows for
-  an embedded `::` in a keyword to to support [garden-style keywords](https://github.com/noprompt/garden)
-  like `:&::before`. This function was transcribed from clj-kondo."
-  [reader]
-  (let [ch (r/read-char reader)]
-    (if-not (reader-impl-utils/whitespace? ch)
-      (let [#?(:clj ^String token :default token) (#'edn/read-token reader :keyword ch)
-            s (reader-impl-commons/parse-symbol token)]
-        (if (and s
-                 ;; (== -1 (.indexOf token "::")) becomes:
-                 (not (zero? (.indexOf token "::"))))
-          (let [#?(:clj ^String ns :default ns) (s 0)
-                #?(:clj ^String name :default name) (s 1)]
-            (if (identical? \: (nth token 0))
-              (reader-impl-errors/throw-invalid reader :keyword token) ; No ::kw in edn.
-              (keyword ns name)))
-          (reader-impl-errors/throw-invalid reader :keyword token)))
-      (reader-impl-errors/throw-single-colon reader))))
-
 (defn parse-keyword
   [#?(:cljs ^not-native reader :default reader)]
-  (u/ignore reader)
-  (if-let [c (r/peek-char reader)]
+  (reader/ignore reader)
+  (if-let [c (reader/peek reader)]
     (if (= c \:)
       (do
-        (r/read-char reader)
+        (reader/next reader)
         (nkeyword/keyword-node
-         (read-keyword reader)
+         (reader/read-keyword reader)
          true))
-      (nkeyword/keyword-node (read-keyword reader)))
-    (u/throw-reader reader "unexpected EOF while reading keyword.")))
+      (nkeyword/keyword-node (reader/read-keyword reader)))
+    (reader/throw-reader reader "unexpected EOF while reading keyword.")))

src/rewrite_clj/parser/namespaced_map.cljc

@@ -1,7 +1,6 @@
 (ns ^:no-doc rewrite-clj.parser.namespaced-map
   (:require [rewrite-clj.node.namespaced-map :as nsmap]
             [rewrite-clj.node.protocols :as node]
-            [rewrite-clj.parser.utils :as u]
             [rewrite-clj.reader :as reader] ))
 
 #?(:clj (set! *warn-on-reflection* true))
@@ -21,18 +20,17 @@
         (recur (conj nodes n))
         [nodes n]))))
 
-
 (defn parse-namespaced-map
   "The caller has parsed up to `#:` and delegates the details to us."
   [reader read-next]
   (reader/ignore reader)
   (let [qualifier-node (parse-qualifier reader)]
     (when (and (not (:auto-resolved? qualifier-node))
                (nil? (:prefix qualifier-node)))
-      (u/throw-reader reader "namespaced map expects a namespace"))
+      (reader/throw-reader reader "namespaced map expects a namespace"))
     (let [[whitespace-nodes map-node] (parse-to-next-elem reader read-next)]
       (when (or (not map-node)
                 (not= :map (node/tag map-node)))
-        (u/throw-reader reader "namespaced map expects a map"))
+        (reader/throw-reader reader "namespaced map expects a map"))
       (nsmap/namespaced-map-node
        (concat [qualifier-node] whitespace-nodes [map-node])))))

src/rewrite_clj/parser/string.cljc

@@ -1,8 +1,7 @@
 (ns ^:no-doc rewrite-clj.parser.string
   (:require [clojure.string :as string]
-            [clojure.tools.reader.reader-types :as r]
             [rewrite-clj.node.stringz :as nstring]
-            [rewrite-clj.parser.utils :as u])
+            [rewrite-clj.reader :as reader])
   #?(:cljs (:import [goog.string StringBuffer])))
 
 #?(:clj (set! *warn-on-reflection* true))
@@ -16,11 +15,11 @@
 
 (defn- read-string-data
   [#?(:cljs ^not-native reader :default reader)]
-  (u/ignore reader)
+  (reader/ignore reader)
   (let [buf (StringBuffer.)]
     (loop [escape? false
            lines []]
-      (if-let [c (r/read-char reader)]
+      (if-let [c (reader/next reader)]
         (cond (and (not escape?) (identical? c \"))
               (flush-into lines buf)
 
@@ -31,7 +30,7 @@
               (do
                 (.append buf c)
                 (recur (and (not escape?) (identical? c \\)) lines)))
-        (u/throw-reader reader "Unexpected EOF while reading string.")))))
+        (reader/throw-reader reader "Unexpected EOF while reading string.")))))
 
 (defn parse-string
   [#?(:cljs ^not-native reader :default reader)]

src/rewrite_clj/parser/utils.cljc

@@ -2,6 +2,9 @@
   (:refer-clojure :exclude [peek next])
   (:require #?@(:clj [[clojure.java.io :as io]])
             [clojure.tools.reader.edn :as edn]
+            [clojure.tools.reader.impl.commons :as reader-impl-commons]
+            [clojure.tools.reader.impl.errors :as reader-impl-errors]
+            [clojure.tools.reader.impl.utils :as reader-impl-utils]
             [clojure.tools.reader.reader-types :as r]
             [rewrite-clj.interop :as interop])
   #?(:cljs (:import [goog.string StringBuffer])
@@ -124,7 +127,12 @@
 (defn peek
   "Peek next char."
   [#?(:cljs ^not-native reader :default reader)]
-  (r/peek-char reader))
+  (let [ch (r/peek-char reader)]
+    ;; compensate for cljs newline normalization in tools reader v1.3.5
+    ;; see https://clojure.atlassian.net/browse/TRDR-65
+    (if (identical? \return ch)
+      \newline
+      ch)))
 
 (defn position
   "Create map of `row-k` and `col-k` representing the current reader position."
@@ -169,13 +177,36 @@
           (if (= n 1) "" "s")))
       vs)))
 
+;;
+;; ## Customizations
+;;
+(defn read-keyword
+  "This customized version of clojure.tools.reader.edn's read-keyword allows for
+  an embedded `::` in a keyword to to support [garden-style keywords](https://github.com/noprompt/garden)
+  like `:&::before`. This function was transcribed from clj-kondo."
+  [reader]
+  (let [ch (r/read-char reader)]
+    (if-not (reader-impl-utils/whitespace? ch)
+      (let [#?(:clj ^String token :default token) (#'edn/read-token reader :keyword ch)
+            s (reader-impl-commons/parse-symbol token)]
+        (if (and s
+                 ;; (== -1 (.indexOf token "::")) becomes:
+                 (not (zero? (.indexOf token "::"))))
+          (let [#?(:clj ^String ns :default ns) (s 0)
+                #?(:clj ^String name :default name) (s 1)]
+            (if (identical? \: (nth token 0))
+              (reader-impl-errors/throw-invalid reader :keyword token) ; No ::kw in edn.
+              (keyword ns name)))
+          (reader-impl-errors/throw-invalid reader :keyword token)))
+      (reader-impl-errors/throw-single-colon reader))))
+
 ;; ## Reader Types
 
 ;;
 ;; clojure.tools.reader (at the time of this writing v1.3.5) does not seem to normalize Windows \r\n newlines
 ;; properly to \n for Clojure 
 ;;
-;; ClojureScript seems to work fine - but note that for peek it can return \r instead of \n. 
+;; ClojureScript seems to work fine - but note that for peek it will return \r for \r\n and \r\f instead of \n. 
 ;;
 ;; see https://clojure.atlassian.net/browse/TRDR-65
 ;;
@@ -185,39 +216,34 @@
 #?(:clj
    (deftype NewlineNormalizingReader
        [rdr
-        ^:unsynchronized-mutable next-char
-        ^:unsynchronized-mutable peeked-char]
+        ^:unsynchronized-mutable read-ahead-char
+        ^:unsynchronized-mutable user-peeked-char]
      r/Reader
      (read-char [_reader]
-       (if peeked-char
-         (let [ch peeked-char]
-           (set! peeked-char nil)
-           ch)
-         (let [ch (or next-char (r/read-char rdr))]
-           (when next-char (set! next-char nil))
-           (cond (identical? \return ch)
-                 (let [next-ch (r/read-char rdr)]
-                   (when (not (identical? \newline next-ch))
-                     (set! next-char next-ch))
-                   \newline)
-
-                 (identical? \formfeed ch)
-                 \newline
-
-                 :else
-                 ch))))
+       (if-let [ch user-peeked-char]
+         (do (set! user-peeked-char nil) ch)
+         (let [ch (or read-ahead-char (r/read-char rdr))]
+           (when read-ahead-char (set! read-ahead-char nil))
+           (if (not (identical? \return ch))
+             ch
+             (let [read-ahead-ch (r/read-char rdr)]
+               (when (not (or (identical? \newline read-ahead-ch)
+                              (identical? \formfeed read-ahead-ch)))
+                 (set! read-ahead-char read-ahead-ch))
+               \newline)))))
 
      (peek-char [reader]
-       (let [ch (or peeked-char (.read-char reader))]
-         (when-not peeked-char (set! peeked-char ch))
-         ch))))
+       (or user-peeked-char
+           (let [ch (.read-char reader)]
+             (set! user-peeked-char ch)
+             ch)))))
 
 #?(:clj
    (defn ^Closeable newline-normalizing-reader
      "Normalizes the following line endings to LF (line feed - 0x0A):
       - LF (remains LF)
       - CRLF (carriage return 0x0D line feed 0x0A)
-      - FF (form feed 0x0C)"
+      - CRFF (carriage return 0x0D form feed 0x0C)"
      [rdr]
      (NewlineNormalizingReader. (r/to-rdr rdr) nil nil)))
 

src/rewrite_clj/reader.cljc

@@ -543,22 +543,20 @@
     ";; comment\r\n(+ 1 2 3)\r\n"
     ";; comment\n(+ 1 2 3)\n"
 
-    "1\r2\r\n3\f4"
+    "1\r2\r\n3\r\f4"
     "1\n2\n3\n4"
 
-    "\n\n\n\n\n"
-    "\n\n\n\n\n"
+    "\n\n\n\n"
+    "\n\n\n\n"
 
     "\r\r\r\r\r"
     "\n\n\n\n\n"
 
-    "\r\n\r\n\r\n\r\n\r\n"
-    "\n\n\n\n\n"
-
-    "\f\f\f\f\f"
-    "\n\n\n\n\n"
+    "\r\n\r\n\r\n\r\n\r\n\r\n"
+    "\n\n\n\n\n\n"
 
-    "\r\n\r\r\f\r\n"
-    "\n\n\n\n\n"))
+    ;1   2 3   4   5 6   7
+    "\r\n\r\r\f\r\n\r\r\n\r"
+    "\n\n\n\n\n\n\n"))