Merge branch 'main' into telkins/objc

Author: trevor-e

.github/workflows/cli_tests.yml

@@ -12,6 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -32,6 +34,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -50,6 +54,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1

README.md

@@ -108,8 +108,33 @@ emerge upload snapshots \
   --project-root /my/awesomeapp/android/repo
 ```
 
-## Building
+## Reaper
 
-This depends on [Tree Sitter](https://tree-sitter.github.io/tree-sitter/) for part of its functionality.
+Experimental support has been added to interactively examine [Reaper](https://docs.emergetools.com/docs/reaper) results and also **delete them from your codebase**.
 
-In order to parse language grammars for Swift and Kotlin, both of which are third-party language grammars, we also depend on [tsdl](https://github.com/stackmystack/tsdl). This downloads and compiles the language grammars into dylibs for us to use.
+Use the `reaper` subcommand to get started, e.g.:
+
+```shell
+emerge reaper --upload-id 40f1dfe7-6c57-47c3-bc52-b621aec0ba8d \
+  --project-root /path/to/your/repo
+```
+
+After which it will prompt you to select classes to delete.
+
+### How it works
+
+Under the hood we are using [Tree Sitter](https://tree-sitter.github.io/tree-sitter/) to parse your source files into an AST which is then used for deletions. There are some obvious limitations to this approach, namely that Tree Sitter is designed for source code editors and only looks at a single file at a time. We are exploring some better long-term approaches but this works well enough for now!
+
+### Supported languages
+
+We currently support the following languages:
+
+- Swift
+- Kotlin
+- Java
+
+Please open an issue if you need an additional language grammar.
+
+### Building
+
+Because many of the language grammars we use are third-party, we have to package them with our CLI tool as shared libraries for distribution. We depend on [tsdl](https://github.com/stackmystack/tsdl) to build the grammars from our `parsers.toml` file.

emerge_cli.gemspec

@@ -30,9 +30,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'async-http', '~> 0.86.0'
+  spec.add_dependency 'CFPropertyList', '~> 2.3', '>= 2.3.2'
   spec.add_dependency 'chunky_png', '~> 1.4.0'
   spec.add_dependency 'dry-cli', '~> 1.2.0'
   spec.add_dependency 'open3', '~> 0.2.1'
+  spec.add_dependency 'ruby-macho', '~> 4.1.0'
   spec.add_dependency 'ruby_tree_sitter', '~> 1.9'
   spec.add_dependency 'tty-prompt', '~> 0.23.1'
   spec.add_dependency 'tty-table', '~> 0.12.0'

lib/commands/global_options.rb

@@ -9,6 +9,8 @@ class GlobalOptions < Dry::CLI::Command
       def before(args)
         log_level = args[:debug] ? ::Logger::DEBUG : ::Logger::INFO
         EmergeCLI::Logger.configure(log_level)
+
+        EmergeCLI::Utils::VersionCheck.new.check_version
       end
     end
   end

lib/commands/order_files/download_order_files.rb

@@ -0,0 +1,77 @@
+require 'dry/cli'
+
+module EmergeCLI
+  module Commands
+    class DownloadOrderFiles < EmergeCLI::Commands::GlobalOptions
+      desc 'Download order files from Emerge'
+
+      option :bundle_id, type: :string, required: true, desc: 'Bundle identifier to download order files for'
+
+      option :api_token, type: :string, required: false,
+                         desc: 'API token for authentication, defaults to ENV[EMERGE_API_TOKEN]'
+
+      option :app_version, type: :string, required: true,
+                           desc: 'App version to download order files for'
+
+      option :unzip, type: :boolean, required: false,
+                     desc: 'Unzip the order file after downloading'
+
+      option :output, type: :string, required: false,
+                      desc: 'Output name for the order file, defaults to bundle_id-app_version.gz'
+
+      EMERGE_ORDER_FILE_URL = 'order-files-prod.emergetools.com'.freeze
+
+      def initialize(network: nil)
+        @network = network
+      end
+
+      def call(**options)
+        @options = options
+        before(options)
+
+        begin
+          api_token = @options[:api_token] || ENV.fetch('EMERGE_API_TOKEN', nil)
+          raise 'API token is required' unless api_token
+
+          raise 'Bundle ID is required' unless @options[:bundle_id]
+          raise 'App version is required' unless @options[:app_version]
+
+          @network ||= EmergeCLI::Network.new(api_token:, base_url: EMERGE_ORDER_FILE_URL)
+          output_name = @options[:output] || "#{@options[:bundle_id]}-#{@options[:app_version]}.gz"
+          output_name = "#{output_name}.gz" unless output_name.end_with?('.gz')
+
+          Sync do
+            request = get_order_file(options[:bundle_id], options[:app_version])
+            response = request.read
+
+            File.write(output_name, response)
+
+            if @options[:unzip]
+              Logger.info 'Unzipping order file...'
+              Zlib::GzipReader.open(output_name) do |gz|
+                File.write(output_name.gsub('.gz', ''), gz.read)
+              end
+            end
+
+            Logger.info 'Order file downloaded successfully'
+          end
+        rescue StandardError => e
+          Logger.error "Failed to download order file: #{e.message}"
+          Logger.error 'Check your parameters and try again'
+          raise e
+        ensure
+          @network&.close
+        end
+      end
+
+      private
+
+      def get_order_file(bundle_id, app_version)
+        @network.get(
+          path: "/#{bundle_id}/#{app_version}",
+          max_retries: 0
+        )
+      end
+    end
+  end
+end

lib/commands/order_files/validate_linkmaps.rb

@@ -0,0 +1,55 @@
+require 'dry/cli'
+require 'cfpropertylist'
+
+module EmergeCLI
+  module Commands
+    class ValidateLinkmaps < EmergeCLI::Commands::GlobalOptions
+      desc 'Validate linkmaps in xcarchive'
+
+      option :path, type: :string, required: true, desc: 'Path to the xcarchive to validate'
+
+      def initialize(network: nil)
+        @network = network
+      end
+
+      def call(**options)
+        @options = options
+        before(options)
+
+        Sync do
+          executable_name = get_executable_name
+          raise 'Executable not found' if executable_name.nil?
+
+          Logger.info "Using executable: #{executable_name}"
+
+          linkmaps_path = File.join(@options[:path], 'Linkmaps')
+          raise 'Linkmaps folder not found' unless File.directory?(linkmaps_path)
+
+          linkmaps = Dir.glob("#{linkmaps_path}/*.txt")
+          raise 'No linkmaps found' if linkmaps.empty?
+
+          executable_linkmaps = linkmaps.select do |linkmap|
+            File.basename(linkmap).start_with?(executable_name)
+          end
+          raise 'No linkmaps found for executable' if executable_linkmaps.empty?
+
+          Logger.info "✅ Found linkmaps for #{executable_name}"
+        end
+      end
+
+      private
+
+      def get_executable_name
+        raise 'Path must be an xcarchive' unless @options[:path].end_with?('.xcarchive')
+
+        app_path = Dir.glob("#{@options[:path]}/Products/Applications/*.app").first
+        info_path = File.join(app_path, 'Info.plist')
+        plist_data = File.read(info_path)
+        plist = CFPropertyList::List.new(data: plist_data)
+        parsed_data = CFPropertyList.native_types(plist.value)
+
+        parsed_data['CFBundleExecutable']
+      end
+    end
+  end
+end

lib/commands/reaper/reaper.rb

@@ -38,8 +38,8 @@ def call(**options)
           project_root = @options[:project_root] || Dir.pwd
 
           Sync do
-            response = @profiler.measure('fetch_dead_code') { fetch_dead_code(@options[:upload_id]) }
-            result = @profiler.measure('parse_dead_code') { DeadCodeResult.new(JSON.parse(response.read)) }
+            all_data = @profiler.measure('fetch_dead_code') { fetch_all_dead_code(@options[:upload_id]) }
+            result = @profiler.measure('parse_dead_code') { DeadCodeResult.new(all_data) }
 
             Logger.info result.to_s
 
@@ -80,11 +80,41 @@ def call(**options)
 
       private
 
-      def fetch_dead_code(upload_id)
-        Logger.info 'Fetching dead code analysis...'
+      def fetch_all_dead_code(upload_id)
+        Logger.info 'Fetching dead code analysis (this may take a while for large codebases)...'
+
+        page = 1
+        combined_data = nil
+
+        loop do
+          response = fetch_dead_code_page(upload_id, page)
+          data = JSON.parse(response.read)
+
+          if combined_data.nil?
+            combined_data = data
+          else
+            combined_data['dead_code'].concat(data.fetch('dead_code', []))
+          end
+
+          current_page = data.dig('pagination', 'current_page')
+          total_pages = data.dig('pagination', 'total_pages')
+
+          break unless current_page && total_pages && current_page < total_pages
+
+          page += 1
+          Logger.info "Fetching page #{page} of #{total_pages}..."
+        end
+
+        combined_data
+      end
+
+      def fetch_dead_code_page(upload_id, page)
         @network.post(
           path: '/deadCode/export',
-          query: { uploadId: upload_id },
+          query: {
+            uploadId: upload_id,
+            page: page
+          },
           headers: { 'Accept' => 'application/json' },
           body: nil
         )

lib/commands/snapshots/validate_app.rb

@@ -0,0 +1,64 @@
+require 'dry/cli'
+require 'json'
+require 'uri'
+require 'yaml'
+require 'cfpropertylist'
+
+module EmergeCLI
+  module Commands
+    module Snapshots
+      class ValidateApp < EmergeCLI::Commands::GlobalOptions
+        desc 'Validate app for snapshot testing [iOS, macOS]'
+
+        # Optional options
+        option :path, type: :string, required: true, desc: 'Path to the app binary or xcarchive'
+
+        # Mangled names are deterministic, no need to demangle them
+        SWIFT_PREVIEWS_MANGLED_NAMES = [
+          '_$s21DeveloperToolsSupport15PreviewRegistryMp',
+          '_$s7SwiftUI15PreviewProviderMp'
+        ].freeze
+
+        def call(**options)
+          @options = options
+          before(options)
+
+          Sync do
+            binary_path = get_binary_path
+            Logger.info "Found binary: #{binary_path}"
+
+            Logger.info "Loading binary: #{binary_path}"
+            macho_parser = MachOParser.new
+            macho_parser.load_binary(binary_path)
+
+            use_chained_fixups, imported_symbols = macho_parser.read_linkedit_data_command
+            bound_symbols = macho_parser.read_dyld_info_only_command
+
+            found = macho_parser.find_protocols_in_swift_proto(use_chained_fixups, imported_symbols, bound_symbols,
+                                                               SWIFT_PREVIEWS_MANGLED_NAMES)
+
+            if found
+              Logger.info '✅ Found SwiftUI previews'
+            else
+              Logger.error '❌ No SwiftUI previews found'
+            end
+            found
+          end
+        end
+
+        private
+
+        def get_binary_path
+          return @options[:path] unless @options[:path].end_with?('.xcarchive')
+          app_path = Dir.glob("#{@options[:path]}/Products/Applications/*.app").first
+          info_path = File.join(app_path, 'Info.plist')
+          plist_data = File.read(info_path)
+          plist = CFPropertyList::List.new(data: plist_data)
+          parsed_data = CFPropertyList.native_types(plist.value)
+
+          File.join(app_path, parsed_data['CFBundleExecutable'])
+        end
+      end
+    end
+  end
+end

lib/emerge_cli.rb

@@ -10,6 +10,9 @@
 require_relative 'commands/config/snapshots/snapshots_ios'
 require_relative 'commands/config/orderfiles/orderfiles_ios'
 require_relative 'commands/reaper/reaper'
+require_relative 'commands/snapshots/validate_app'
+require_relative 'commands/order_files/download_order_files'
+require_relative 'commands/order_files/validate_linkmaps'
 
 require_relative 'reaper/ast_parser'
 require_relative 'reaper/code_deleter'
@@ -22,10 +25,10 @@
 require_relative 'utils/network'
 require_relative 'utils/profiler'
 require_relative 'utils/project_detector'
+require_relative 'utils/macho_parser'
+require_relative 'utils/version_check'
 
 require 'dry/cli'
-require 'pry'
-require 'pry-byebug'
 
 module EmergeCLI
   extend Dry::CLI::Registry
@@ -44,6 +47,15 @@ module EmergeCLI
   end
 
   register 'reaper', Commands::Reaper
+
+  register 'snapshots' do |prefix|
+    prefix.register 'validate-app-ios', Commands::Snapshots::ValidateApp
+  end
+
+  register 'order-files' do |prefix|
+    prefix.register 'download', Commands::DownloadOrderFiles
+    prefix.register 'validate-linkmaps', Commands::ValidateLinkmaps
+  end
 end
 
 # By default the log level is INFO, but can be overridden by the --debug flag