Adjust ArchGen to account for json references

Author: Derek Hower

backends/arch_gen/lib/arch_gen.rb

@@ -52,7 +52,7 @@ def gen_params_schema
     @implemented_extensions.each do |ext|
       ext_name = ext["name"]
       gen_ext_path = @gen_dir / "arch" / "ext" / "#{ext_name}.yaml"
-      ext_yaml = YamlLoader.load gen_ext_path.to_s
+      ext_yaml = YAML.load_file gen_ext_path.to_s
       unless ext_yaml[ext_name]["params"].nil?
         ext_yaml[ext_name]["params"].each do |param_name, param_data|
           schema["properties"]["params"]["required"] << param_name
@@ -146,7 +146,7 @@ def params_extra_validation
     @implemented_extensions.each do |ext|
       ext_name = ext["name"]
       gen_ext_path = @gen_dir / "arch" / "ext" / "#{ext_name}.yaml"
-      ext_yaml = YamlLoader.load gen_ext_path.read
+      ext_yaml = YAML.load_file gen_ext_path.read
       unless ext_yaml[ext_name]["params"].nil?
         ext_yaml[ext_name]["params"].each do |param_name, param_data|
           next unless param_data.key?("extra_validation")
@@ -545,99 +545,6 @@ def arch_overlay_path_for(type, name)
   def schema_path_for(type) = Validator::SCHEMA_PATHS[type]
   private :schema_path_for
 
-  # Render a architecture definition file and save it to gen_dir / .rendered_arch
-  #
-  # Will not re-render if rendered file already exists and sources have not changed
-  #
-  # @param type [Symbol] Type of the object (@see Validator::SCHEMA_PATHS)
-  # @param name [#to_s] Name of the object
-  # @param extra_env [Hash,NilObject] Optional hash with extra enviornment variables for the render
-  # @return [Pathname,nil] Path to generated file, or nil if there is no (valid) definition for type,name
-  def gen_rendered_arch_def(type, name, extra_env = {})
-    gen_path = @gen_dir / ".rendered_arch" / type.to_s / "#{name}.yaml"
-
-    source_path = arch_path_for(type, name)
-    return nil if source_path.nil?
-
-    schema_path = schema_path_for(type)
-
-    if gen_path.exist?
-      # this already exists...see if we need to regenerate it
-      dep_mtime = [File.mtime(__FILE__), source_path.mtime, schema_path.mtime].max
-      return gen_path if gen_path.mtime >= dep_mtime # no update needed
-    end
-
-    trace "Rendering architecture file for #{type}:#{name}"
-
-    # render the source template
-    current_env = env.clone
-    extra_env&.each { |k, v| current_env.define_singleton_method(k) { v } }
-    rendered_def = Tilt["erb"].new(source_path, trim: "-").render(current_env)
-
-    # see if the rendering was empty, meaning that the def isn't valid in this config
-    return nil if rendered_def.nil?
-
-    # write the object
-    FileUtils.mkdir_p gen_path.dirname
-    File.write(gen_path, rendered_def)
-
-    # verify
-    begin
-      @validator.validate_str(rendered_def, type:)
-    rescue Validator::SchemaValidationError => e
-      warn "#{type} definition in #{source_path} did not validate"
-      raise e
-    end
-
-    def_obj = YamlLoader.load(rendered_def)
-
-    raise "#{type} name ('#{name}') must match key in defintion ('#{def_obj.keys[0]}')" if name.to_s != def_obj.keys[0]
-
-    # return path to generated file
-    gen_path
-  end
-  private :gen_rendered_arch_def
-
-  # Render a architecture overlay definition file and save it to gen_dir / .rendered_overlay_arch
-  #
-  # Will not re-render if rendered file already exists and sources have not changed
-  #
-  # @param type [Symbol] Type of the object (@see Validator::SCHEMA_PATHS)
-  # @param name [#to_s] Name of the object
-  # @param extra_env [Hash,NilObject] Optional hash with extra enviornment variables for the render
-  # @return [Pathname] Path to generated file
-  def gen_rendered_arch_overlay_def(type, name, extra_env = {})
-    gen_path = @gen_dir / ".rendered_overlay_arch" / type.to_s / "#{name}.yaml"
-
-    source_path = arch_overlay_path_for(type, name)
-    return nil if source_path.nil?
-
-    if gen_path.exist?
-      # this already exists...see if we need to regenerate it
-      dep_mtime = [File.mtime(__FILE__), source_path.mtime].max
-      return gen_path if gen_path.mtime >= dep_mtime # no update needed
-    end
-
-    trace "Rendering overlay file for #{type}:#{name}"
-
-    # render the source template
-    current_env = env.clone
-    extra_env&.each { |k, v| current_env.define_singleton_method(k) { v } }
-    rendered_def = Tilt["erb"].new(source_path, trim: "-").render(current_env)
-
-    def_obj = YamlLoader.load(rendered_def)
-
-    raise "#{type} name (#{name}) must match key in defintion (#{def_obj.keys[0]})" if name.to_s != def_obj.keys[0]
-
-    # write the object
-    FileUtils.mkdir_p gen_path.dirname
-    File.write(gen_path, YAML.dump(def_obj))
-
-    # return path to generated file
-    gen_path
-  end
-  private :gen_rendered_arch_overlay_def
-
   # generate a merged definition from rendered arch and overlay, and write it to gen / .merged_arch
   #
   # Skips if gen file already exists and sources are older
@@ -665,10 +572,12 @@ def gen_merged_def(type, arch_path, overlay_path)
     FileUtils.mkdir_p merged_path.dirname
     if overlay_path.nil?
       # no overlay, just copy arch
-      FileUtils.cp arch_path, merged_path
+      merged_path.write YAML.dump(YamlLoader.load(arch_path))
+      # FileUtils.cp arch_path, merged_path
     elsif arch_path.nil?
       # no arch, overlay is arch
-      FileUtils.cp overlay_path, merged_path
+      merged_path.write YAML.dump(YamlLoader.load(overlay_path))
+      # FileUtils.cp overlay_path, merged_path
     else
       # arch and overlay, do the merge
       arch_obj = YamlLoader.load(arch_path)
@@ -695,8 +604,8 @@ def gen_merged_def(type, arch_path, overlay_path)
   # @param csr_name [#to_s] CSR name
   # @param extra_env [Hash] Extra enviornment variables to be used when parsing the CSR definition template
   def maybe_add_csr(csr_name, extra_env = {})
-    arch_path         = arch_path_for(:csr, csr_name) # gen_rendered_arch_def(:csr, csr_name, extra_env)
-    arch_overlay_path = arch_overlay_path_for(:csr, csr_name) # gen_rendered_arch_overlay_def(:csr, csr_name, extra_env)
+    arch_path         = arch_path_for(:csr, csr_name)
+    arch_overlay_path = arch_overlay_path_for(:csr, csr_name)
 
     # return immediately if this CSR isn't defined in this config
     raise "No arch or overlay for sr #{csr_name}" if arch_path.nil? && arch_overlay_path.nil?
@@ -721,7 +630,7 @@ def maybe_add_csr(csr_name, extra_env = {})
       end
 
     # get the csr data (not including the name key), which is redundant at this point
-    csr_data = YamlLoader.load(merged_path)[csr_name]
+    csr_data = YAML.load_file(merged_path)[csr_name]
     csr_data["name"] = csr_name
     csr_data["fields"].each { |n, f| f["name"] = n }
     csr_data["__source"] = og_path.to_s
@@ -796,15 +705,15 @@ def all_known_exts
   end
 
   def maybe_add_ext(ext_name)
-    arch_path         = arch_path_for(:ext, ext_name) # gen_rendered_arch_def(:ext, ext_name)
-    arch_overlay_path = arch_overlay_path_for(:ext, ext_name) # gen_rendered_arch_overlay_def(:ext, ext_name)
+    arch_path         = arch_path_for(:ext, ext_name)
+    arch_overlay_path = arch_overlay_path_for(:ext, ext_name)
 
     # return immediately if this ext isn't defined
     return if arch_path.nil? && arch_overlay_path.nil?
 
     merged_path = gen_merged_def(:ext, arch_path, arch_overlay_path)
 
-    yaml_contents = YamlLoader.load(merged_path)
+    yaml_contents = YAML.load_file(merged_path)
     raise "In #{merged_path}, key does not match file name" unless yaml_contents.key?(ext_name)
 
     ext_obj = yaml_contents[ext_name]
@@ -940,8 +849,8 @@ def possible_xlens
   # @param inst_name [#to_s] instruction name
   # @param extra_env [Hash] Extra options to add into the rendering enviornment
   def maybe_add_inst(inst_name, extra_env = {})
-    arch_path         = arch_path_for(:inst, inst_name) # gen_rendered_arch_def(:inst, inst_name, extra_env)
-    arch_overlay_path = arch_overlay_path_for(:inst, inst_name) # gen_rendered_arch_overlay_def(:inst, inst_name, extra_env)
+    arch_path         = arch_path_for(:inst, inst_name)
+    arch_overlay_path = arch_overlay_path_for(:inst, inst_name)
 
     # return immediately if inst isn't defined in this config
     raise "No arch or overlay for instruction #{inst_name}" if arch_path.nil? && arch_overlay_path.nil?
@@ -966,7 +875,7 @@ def maybe_add_inst(inst_name, extra_env = {})
       end
 
     # get the inst data (not including the name key), which is redundant at this point
-    inst_data = YamlLoader.load(merged_path)
+    inst_data = YAML.load_file(merged_path)
     raise "The first and only key of #{arch_path} must be '#{inst_name}" unless inst_data.key?(inst_name)
     inst_data = inst_data[inst_name]
 

lib/validate.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "yaml_loader"
+
 require "date"
 require "json"
 require "json_schemer"
@@ -256,7 +258,7 @@ def validate_instruction_encoding(inst_name, encoding)
   # @param path [Pathname] Path to an instruction YAML document
   # @raise [ValidateError] if there is a problem with the instruction defintion
   def validate_instruction(path)
-    obj = YAML.load_file(path)
+    obj = YamlLoader.load(path)
     raise "Invalid instruction definition: #{obj}" unless obj.is_a?(Hash)
 
     inst_name = path.basename('.yaml').to_s

lib/yaml_loader.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "pathname"
 require "yaml"
 
 # loads a YAML file and expands any $ref/$mref references
@@ -22,8 +23,11 @@ def self.expand(filename, obj, yaml_opts = {})
 
         if key == "$mref"
           relative_path = value.split("#")[0]
-          target_filename = File.realpath(File.join(File.dirname(filename), relative_path))
-          raise "#{target_filename} does not exist" unless File.exist?(target_filename)
+          target_filename = File.realpath(File.join(filename.dirname, relative_path))
+
+          unless File.exist?(target_filename)
+            raise "While locating $mref in #{filename}, #{target_filename} does not exist"
+          end
 
           target_obj = YamlLoader.load(target_filename, yaml_opts)
           target_obj = target_obj.dig(*value.split("#/")[1].split("/"))
@@ -44,14 +48,15 @@ def self.expand(filename, obj, yaml_opts = {})
   # @param yaml_opts [Hash] options to pass to YAML.load_file
   # @return [Object] the loaded YAML file
   def self.load(filename, yaml_opts = {})
-    raise ArgumentError, "Cannot find file #{filename}" unless File.exist?(filename)
+    filename = Pathname.new(filename)
+    raise ArgumentError, "Cannot find file #{filename}" unless filename.exist?
 
-    filename = File.realpath(filename)
+    filename = filename.realpath
     return @cache[filename] if @cache.key?(filename)
 
     obj = YAML.load_file(filename, **yaml_opts)
     obj = expand(filename, obj, yaml_opts) if obj.is_a?(Hash)
 
-    @cache[filename] = obj
+    # @cache[filename] = obj
   end
 end