[API] Generator: Update code generation for merged specs

Author: picandocodigo

elasticsearch-api/utils/Gemfile

@@ -18,10 +18,11 @@
 source 'https://rubygems.org'
 
 gem 'activesupport'
-gem 'rest-client'
+gem 'byebug'
 gem 'coderay'
-gem 'multi_json'
-gem 'thor'
 gem 'json'
+gem 'multi_json'
 gem 'pry'
+gem 'rest-client'
 gem 'rubocop'
+gem 'thor'

elasticsearch-api/utils/thor/generate_source.rb

@@ -66,11 +66,10 @@ def generate
 
       def __generate_source(api)
         @current_api = api
-        @input = FilesHelper.input_dir(api)
         @output = FilesHelper.output_dir(api)
 
         FilesHelper.files(api).each do |filepath|
-          @path = Pathname(@input.join(filepath))
+          @path = Pathname(filepath)
           @json = MultiJson.load(File.read(@path))
           @spec = @json.values.first
           say_status 'json', @path, :yellow

elasticsearch-api/utils/thor/generator/files_helper.rb

@@ -21,36 +21,52 @@
 module Elasticsearch
   module API
     module FilesHelper
-      OSS_SRC_PATH   = '../../../../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
+      PROJECT_PATH = File.join(File.dirname(__FILE__), '..')
+      SRC_PATH   = File.join(PROJECT_PATH, '..', '..', '..', 'tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/')
       OSS_OUTPUT_DIR = '../../elasticsearch-api/lib/elasticsearch/api/actions'.freeze
-
-      XPACK_SRC_PATH   = '../../../../../tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/api'.freeze
       XPACK_OUTPUT_DIR = '../../elasticsearch-xpack/lib/elasticsearch/xpack/api/actions'.freeze
 
-      # Path to directory with JSON API specs
-      def self.input_dir(api)
-        input_dir = if api == :xpack
-                      File.expand_path(XPACK_SRC_PATH, __FILE__)
-                    else
-                      File.expand_path(OSS_SRC_PATH, __FILE__)
-                    end
-        Pathname(input_dir)
+      TESTS_DIRECTORIES = {
+        oss: "#{PROJECT_PATH}/../../../tmp/rest-api-spec/test/free",
+        xpack: "#{PROJECT_PATH}/../../../tmp/rest-api-spec/test/platinum"
+      }.freeze
+
+      # Only get JSON files and remove hidden files
+      def self.files(api)
+        json_files = if api == :xpack
+                       xpack_files
+                     else
+                       Dir.entries(SRC_PATH) - xpack_files
+                     end
+        json_files.reject do |file|
+          File.extname(file) != '.json' ||
+            File.basename(file) == '_common.json'
+        end.map { |file| "#{SRC_PATH}#{file}" }
+      end
+
+      XPACK_ENDPOINTS = [
+        'autoscaling', 'cross_cluster_replication', 'ccr', 'data_frame_transform_deprecated',
+        'enrich', 'eql', 'ilm', 'logstash', 'migration', 'watcher', 'slm'
+      ]
+      XPACK_ENDPOINTS_REGEXP = /data_stream|ml_|reload_search_analyzers|transform/
+
+      def self.xpack_files
+        xpack_tests = Dir.entries(TESTS_DIRECTORIES[:xpack])
+        Dir.entries(SRC_PATH).map do |entry|
+          filename = entry.split('.').first
+          if xpack_tests.include?(filename) ||
+                                  XPACK_ENDPOINTS.include?(filename) ||
+                                  entry.match?(XPACK_ENDPOINTS_REGEXP)
+            entry
+          end
+        end.compact
       end
 
       # Path to directory to copy generated files
       def self.output_dir(api)
         api == :xpack ? Pathname(XPACK_OUTPUT_DIR) : Pathname(OSS_OUTPUT_DIR)
       end
 
-      # Only get JSON files and remove hidden files
-      def self.files(api)
-        Dir.entries(input_dir(api).to_s).reject do |f|
-          f.start_with?('.') ||
-            f.start_with?('_') ||
-            File.extname(f) != '.json'
-        end
-      end
-
       def self.gem_version
         regex = /([0-9]{1,2}\.[0-9x]{1,2})/
         Elasticsearch::API::VERSION.match(regex)[0]