Implement multipass analysis

Closes #380. This changes how Resyntax works by making it produce a single "analysis" object describing all of the multiple rounds of changes to apply to analyzed code before actually modifying any files. The CLI layer is changed into a pure frontend to the analysis API, with none of the multipass logic implemented in the CLI like it was previously.

Author: jackfirth

cli.rkt

@@ -4,8 +4,10 @@
 (require fancy-app
          json
          racket/cmdline
+         racket/file
          racket/format
          racket/hash
+         (except-in racket/list range)
          racket/logging
          racket/match
          racket/path
@@ -14,6 +16,7 @@
          rebellion/collection/entry
          rebellion/collection/hash
          rebellion/collection/list
+         rebellion/collection/multiset
          rebellion/collection/range-set
          rebellion/collection/vector/builder
          rebellion/streaming/reducer
@@ -218,15 +221,15 @@ For help on these, use 'analyze --help' or 'fix --help'."
 
 (define (resyntax-analyze-run)
   (define options (resyntax-analyze-parse-command-line))
-  (define files (file-groups-resolve (resyntax-analyze-options-targets options)))
-  (printf "resyntax: --- analyzing code ---\n")
+  (define sources (file-groups-resolve (resyntax-analyze-options-targets options)))
+  (define analysis
+    (resyntax-analyze-all sources
+                          #:suite (resyntax-analyze-options-suite options)
+                          #:max-passes 1))
   (define results
-    (transduce files
-               (append-mapping
-                (λ (portion)
-                  (resyntax-analyze (file-source (file-portion-path portion))
-                                    #:suite (resyntax-analyze-options-suite options)
-                                    #:lines (file-portion-lines portion))))
+    (transduce (resyntax-analysis-all-results analysis)
+               (append-mapping in-hash-values)
+               (append-mapping refactoring-result-set-results)
                #:into into-list))
 
   (define (display-results)
@@ -244,7 +247,7 @@ For help on these, use 'analyze --help' or 'fix --help'."
                  (string-indent (~a old-code) #:amount 2)
                  (string-indent (~a new-code) #:amount 2)))]
       [(== github-pull-request-review)
-       (define req (refactoring-results->github-review results #:file-count (length files)))
+       (define req (refactoring-results->github-review results #:file-count (length sources)))
        (write-json (github-review-request-jsexpr req))]))
 
   (match (resyntax-analyze-options-output-destination options)
@@ -259,191 +262,91 @@ For help on these, use 'analyze --help' or 'fix --help'."
 (define (resyntax-fix-run)
   (define options (resyntax-fix-parse-command-line))
   (define output-format (resyntax-fix-options-output-format options))
-  (match output-format
-    [(== git-commit-message)
-     (display "This is an automated change generated by Resyntax.\n\n")]
-    [_ (void)])
-  (define files
-    (transduce (file-groups-resolve (resyntax-fix-options-targets options))
-               (indexing file-portion-path)
-               (grouping into-list)
-               #:into into-hash))
+  (define sources (file-groups-resolve (resyntax-fix-options-targets options)))
   (define max-modified-files (resyntax-fix-options-max-modified-files options))
   (define max-modified-lines (resyntax-fix-options-max-modified-lines options))
-  (define results-by-path
-    (for/fold ([all-results (hash)]
-               [files files]
-               [max-fixes (resyntax-fix-options-max-fixes options)]
-               [lines-to-analyze-by-file (hash)]
-               #:result all-results)
-              ([pass-number (in-inclusive-range 1 (resyntax-fix-options-max-pass-count options))]
-               #:do [(define pass-results
-                       (resyntax-fix-run-one-pass options files
-                                                  #:lines lines-to-analyze-by-file
-                                                  #:max-fixes max-fixes
-                                                  #:max-modified-files max-modified-files
-                                                  #:max-modified-lines max-modified-lines
-                                                  #:pass-number pass-number))
-                     (define pass-fix-count
-                       (for/sum ([(_ results) (in-hash pass-results)])
-                         (length results)))
-                     (define pass-modified-file-count (hash-count pass-results))
-                     (define new-max-fixes (- max-fixes pass-fix-count))]
-               #:break (hash-empty? pass-results)
-               #:final (zero? new-max-fixes))
-      (define new-files (hash-filter-keys files (hash-has-key? pass-results _)))
-      (define new-lines-to-analyze
-        (for/hash ([(path results) (in-hash pass-results)])
-          (values path
-                  (transduce results
-                             (mapping refactoring-result-modified-line-range)
-                             (filtering nonempty-range?)
-                             #:into (into-range-set natural<=>)))))
-      (values (hash-union all-results pass-results #:combine append)
-              new-files
-              new-max-fixes
-              new-lines-to-analyze)))
+  (define analysis
+    (resyntax-analyze-all sources
+                          #:suite (resyntax-fix-options-suite options)
+                          #:max-fixes (resyntax-fix-options-max-fixes options)
+                          #:max-passes (resyntax-fix-options-max-pass-count options)
+                          #:max-modified-sources max-modified-files
+                          #:max-modified-lines max-modified-lines))
+  (resyntax-analysis-write-file-changes! analysis)
   (match output-format
-    [(== plain-text) (printf "resyntax: --- summary ---\n")]
-    [(== git-commit-message) (printf "## Summary\n\n")])
-  (define total-fixes
-    (for/sum ([(_ results) (in-hash results-by-path)])
-      (length results)))
-  (define total-files (hash-count results-by-path))
+    [(== git-commit-message)
+     (resyntax-fix-print-git-commit-message analysis)]
+    [(== plain-text)
+     (resyntax-fix-print-plain-text-summary analysis)]))
+
+
+(define (resyntax-fix-print-git-commit-message analysis)
+  (display "This is an automated change generated by Resyntax.\n\n")
+  (for ([pass-results (resyntax-analysis-all-results analysis)]
+        [pass-number (in-naturals 1)])
+    (unless (hash-empty? pass-results)
+      (printf "#### Pass ~a\n\n" pass-number))
+    (for ([(source result-set) (in-hash pass-results)])
+      (define result-count (length (refactoring-result-set-results result-set)))
+      (define fix-string (if (> result-count 1) "fixes" "fix"))
+      ;; For a commit message, we always use a relative path since we're likely running inside
+      ;; some CI runner. Additionally, we make the path a link to the corresponding file at HEAD,
+      ;; since making file paths clickable is pleasant.
+      (define relative-path (find-relative-path (current-directory) (source-path source)))
+      (define repo-head-path (format "../blob/HEAD/~a" relative-path))
+      (printf "Applied ~a ~a to [`~a`](~a)\n\n"
+              result-count fix-string relative-path repo-head-path)
+      (for ([result (in-list (refactoring-result-set-results result-set))])
+        (define line (refactoring-result-original-line result))
+        (define rule (refactoring-result-rule-name result))
+        (define message (refactoring-result-message result))
+        (printf "  * Line ~a, `~a`: ~a\n" line rule message))
+      (newline)))
+  (printf "## Summary\n\n")
+  (define total-fixes (resyntax-analysis-total-fixes analysis))
+  (define total-files (resyntax-analysis-total-sources-modified analysis))
   (define fix-counts-by-rule
-    (transduce (hash-values results-by-path)
-               (append-mapping values)
-               (indexing refactoring-result-rule-name)
-               (grouping into-count)
+    (transduce (in-hash-entries (multiset-frequencies (resyntax-analysis-rules-applied analysis)))
                (sorting #:key entry-value #:descending? #true)
                #:into into-list))
   (define issue-string (if (> total-fixes 1) "issues" "issue"))
   (define file-string (if (> total-files 1) "files" "file"))
-  (define summary-message
-    (if (zero? total-fixes)
-        "No issues found.\n"
-        (format "Fixed ~a ~a in ~a ~a.\n\n" total-fixes issue-string total-files file-string)))
-  (match output-format
-    [(== plain-text) (printf "\n  ~a" summary-message)]
-    [(== git-commit-message) (printf summary-message)])
+  (if (zero? total-fixes)
+      (printf "No issues found.\n")
+      (printf "Fixed ~a ~a in ~a ~a.\n\n" total-fixes issue-string total-files file-string))
   (for ([rule+count (in-list fix-counts-by-rule)])
     (match-define (entry rule count) rule+count)
     (define occurrence-string (if (> count 1) "occurrences" "occurrence"))
-    (define rule-string
-      (match output-format
-        [(== plain-text) rule]
-        [(== git-commit-message) (format "`~a`" rule)]))
-    (printf "  * Fixed ~a ~a of ~a\n" count occurrence-string rule-string))
+    (printf "  * Fixed ~a ~a of `~a`\n" count occurrence-string rule))
   (unless (zero? total-fixes)
     (newline)))
 
 
-(define (resyntax-fix-run-one-pass options files
-                                   #:lines lines-to-analyze-by-file
-                                   #:max-fixes max-fixes
-                                   #:max-modified-files max-modified-files
-                                   #:max-modified-lines max-modified-lines
-                                   #:pass-number pass-number)
-  (define output-format (resyntax-fix-options-output-format options))
-  (match output-format
-    [(== plain-text)
-     (unless (equal? pass-number 1)
-       (printf "resyntax: --- pass ~a ---\n" pass-number))
-     (printf "resyntax: --- analyzing code ---\n")]
-    [_ (void)])
-  (define all-results
-    (transduce (in-hash-entries files) ; entries with file path keys and lists of file-portion? values
-
-               ;; The following steps perform a kind of layered shuffle: the files to refactor are
-               ;; shuffled such that files in the same directory remain together. When combined with
-               ;; the #:max-modified-files argument, this makes Resyntax prefer to refactor closely
-               ;; related files instead of selecting arbitrary unrelated files from across an entire
-               ;; codebase. This limits potential for merge conflicts and makes changes easier to
-               ;; review, since it's more likely the refactored files will have shared context.
-
-               ; key by directory
-               (indexing (λ (e) (simple-form-path (build-path (entry-key e) 'up))))
-
-               ; group by key and shuffle within each group
-               (grouping (into-transduced (shuffling) #:into into-list))
-
-               ; shuffle groups
-               (shuffling)
-
-               ; ungroup and throw away directory
-               (append-mapping entry-value)
-
-               ;; Now the stream contains exactly what it did before the above steps, but shuffled in
-               ;; a convenient manner.
-               
-               (append-mapping entry-value) ; throw away the file path, we don't need it anymore
-               (mapping (filter-file-portion _ lines-to-analyze-by-file))
-               (append-mapping
-                (λ (portion)
-                  (resyntax-analyze (file-source (file-portion-path portion))
-                                    #:suite (resyntax-fix-options-suite options)
-                                    #:lines (file-portion-lines portion))))
-               (limiting max-modified-lines
-                         #:by (λ (result)
-                                (define replacement (refactoring-result-line-replacement result))
-                                (add1 (- (line-replacement-original-end-line replacement)
-                                         (line-replacement-start-line replacement)))))
-               (if (equal? max-fixes +inf.0) (transducer-pipe) (taking max-fixes))
-               (if (equal? max-modified-files +inf.0)
-                   (transducer-pipe)
-                   (transducer-pipe
-                    (indexing
-                     (λ (result)
-                       (syntax-replacement-source (refactoring-result-syntax-replacement result))))
-                    (grouping into-list)
-                    (taking max-modified-files)
-                    (append-mapping entry-value)))
-               #:into into-list))
-  (define results-by-path
-    (transduce
-     all-results
-     (indexing
-      (λ (result)
-        (file-source-path
-         (syntax-replacement-source (refactoring-result-syntax-replacement result)))))
-     (grouping (into-transduced (sorting #:key refactoring-result-original-line) #:into into-list))
-     #:into into-hash))
-  (match output-format
-    [(== plain-text) (printf "resyntax: --- fixing code ---\n")]
-    [(== git-commit-message)
-     (unless (hash-empty? results-by-path)
-       (printf "#### Pass ~a\n\n" pass-number))])
-  (for ([(path results) (in-hash results-by-path)])
-    (define result-count (length results))
-    (define fix-string (if (> result-count 1) "fixes" "fix"))
-    (match output-format
-      [(== plain-text)
-       (printf "resyntax: applying ~a ~a to ~a\n\n" result-count fix-string path)]
-      [(== git-commit-message)
-       ;; For a commit message, we always use a relative path since we're likely running inside
-       ;; some CI runner. Additionally, we make the path a link to the corresponding file at HEAD,
-       ;; since making file paths clickable is pleasant.
-       (define relative-path (find-relative-path (current-directory) path))
-       (define repo-head-path (format "../blob/HEAD/~a" relative-path))
-       (printf "Applied ~a ~a to [`~a`](~a)\n\n"
-               result-count fix-string relative-path repo-head-path)])
-    (for ([result (in-list results)])
-      (define line (refactoring-result-original-line result))
-      (define rule (refactoring-result-rule-name result))
-      (define message (refactoring-result-message result))
-      (match output-format
-        [(== plain-text) (printf "  * [line ~a] ~a: ~a\n" line rule message)]
-        [(== git-commit-message) (printf "  * Line ~a, `~a`: ~a\n" line rule message)]))
-    (refactor! results)
-    (newline))
-  results-by-path)
-
-
-(define (filter-file-portion portion lines-by-path)
-  (define path (file-portion-path portion))
-  (define lines (file-portion-lines portion))
-  (define ranges-to-remove (range-set-complement (hash-ref lines-by-path path all-lines)))
-  (file-portion path (range-set-remove-all lines ranges-to-remove)))
+(define (resyntax-fix-print-plain-text-summary analysis)
+  (printf "resyntax: --- summary ---\n\n")
+  (define total-fixes (resyntax-analysis-total-fixes analysis))
+  (define total-files (resyntax-analysis-total-sources-modified analysis))
+  (define message
+    (cond
+      [(zero? total-fixes) "No issues found."]
+      [(equal? total-fixes 1) "Fixed 1 issue in 1 file."]
+      [(equal? total-files 1) (format "Fixed ~a issues in 1 file." total-fixes)]
+      [else (format "Fixed ~a issues in ~a files." total-fixes total-files)]))
+  (printf "  ~a\n\n" message)
+  (define rules-applied (resyntax-analysis-rules-applied analysis))
+  (transduce (in-hash-entries (multiset-frequencies rules-applied))
+             (sorting #:key entry-value #:descending? #true)
+             (mapping
+              (λ (e)
+                (match-define (entry rule-name rule-fixes) e)
+                (define message
+                  (if (equal? rule-fixes 1)
+                      (format "Fixed 1 occurrence of ~a" rule-name)
+                      (format "Fixed ~a occurrences of ~a" rule-fixes rule-name)))
+                (format "  * ~a\n" message)))
+             #:into (into-for-each display))
+  (when (positive? total-fixes)
+    (newline)))
 
 
 (module+ main