Add $mref to inst variables (#228)

* Add $mref to inst variables

Also enhances arch_gen to expand references.

Supports $ref (JSON Reference) and $mref (merged ref, doesn't ignore other keys in the object)

Difference:

(in something.yaml)
key:
  ref_extra: Hello!

obj:
  $ref: path/to/something.yaml#/key
  extra: Hi!

=> obj = { ref_extra: "Hello!" }

obj:
  $mref: path/to/something.yaml#/key
  extra: Hi!

=> obj = { extra: "Hi!", ref_extra: "Hello!" }

* Adjust ArchGen to account for json references

* bug fix

* Add YamlLoader test, support for multiple

Author: dhower-qc

Rakefile

@@ -55,8 +55,20 @@ namespace :serve do
   end
 end
 
-Minitest::TestTask.create :idl_test do |t|
+desc "Run the IDL compiler test suite"
+task :idl_test do
+  t = Minitest::TestTask.new(:lib_test)
   t.test_globs = ["#{$root}/lib/idl/tests/test_*.rb"]
+  t.process_env
+  ruby t.make_test_cmd
+end
+
+desc "Run the Ruby library test suite"
+task :lib_test do
+  t = Minitest::TestTask.new(:lib_test)
+  t.test_globs = ["#{$root}/lib/test/test_*.rb"]
+  t.process_env
+  ruby t.make_test_cmd
 end
 
 desc "Clean up all generated files"
@@ -269,6 +281,7 @@ desc <<~DESC
 DESC
 task :regress do
   Rake::Task["idl_test"].invoke
+  Rake::Task["lib_test"].invoke
   Rake::Task["validate"].invoke
   ENV["MANUAL_NAME"] = "isa"
   ENV["VERSIONS"] = "all"

arch/common/inst_variable_types.yaml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../schemas/inst_variable_metadatas.json
+---
+
+itype_imm:
+  location: 31-20

arch/inst/I/addi.yaml

@@ -9,7 +9,7 @@ addi:
     match: -----------------000-----0010011
     variables:
     - name: imm
-      location: 31-20
+      $mref: ../../common/inst_variable_types.yaml#/itype_imm
     - name: rs1
       location: 19-15
     - name: rd

backends/arch_gen/lib/arch_gen.rb

@@ -6,7 +6,6 @@
 require "rake/application"
 require "rubygems/requirement"
 require "tilt"
-require "yaml"
 
 require_relative "#{$lib}/validate"
 require_relative "#{$lib}/arch_def"
@@ -53,7 +52,7 @@ def gen_params_schema
     @implemented_extensions.each do |ext|
       ext_name = ext["name"]
       gen_ext_path = @gen_dir / "arch" / "ext" / "#{ext_name}.yaml"
-      ext_yaml = YAML.safe_load gen_ext_path.read
+      ext_yaml = YAML.load_file gen_ext_path.to_s
       unless ext_yaml[ext_name]["params"].nil?
         ext_yaml[ext_name]["params"].each do |param_name, param_data|
           schema["properties"]["params"]["required"] << param_name
@@ -92,7 +91,7 @@ def initialize(config_name)
     raise "Validation failed" if @cfg_impl_ext.nil?
 
     cfg_opts_path = @cfg_dir / "cfg.yaml"
-    @cfg_opts = YAML.load_file(cfg_opts_path)
+    @cfg_opts = YamlLoader.load(cfg_opts_path)
     raise "Validation failed" if @cfg_opts.nil?
     raise "Validation failed: bad type" unless ["partially configured", "fully configured"].include?(@cfg_opts["type"])
 
@@ -147,7 +146,7 @@ def params_extra_validation
     @implemented_extensions.each do |ext|
       ext_name = ext["name"]
       gen_ext_path = @gen_dir / "arch" / "ext" / "#{ext_name}.yaml"
-      ext_yaml = YAML.safe_load gen_ext_path.read
+      ext_yaml = YAML.load_file gen_ext_path.to_s
       unless ext_yaml[ext_name]["params"].nil?
         ext_yaml[ext_name]["params"].each do |param_name, param_data|
           next unless param_data.key?("extra_validation")
@@ -275,63 +274,63 @@ def implemented_extensions
   #
   def gen_arch_def
     csr_hash = Dir.glob(@gen_dir / "arch" / "csr" / "**" / "*.yaml").map do |f|
-      csr_obj = YAML.load_file(f)
+      csr_obj = YamlLoader.load(f)
       csr_name = csr_obj.keys[0]
       [csr_name, csr_obj[csr_name]]
     end.to_h
     inst_hash = Dir.glob(@gen_dir / "arch" / "inst" / "**" / "*.yaml").map do |f|
-      inst_obj = YAML.load_file(f)
+      inst_obj = YamlLoader.load(f)
       inst_name = inst_obj.keys[0]
       [inst_name, inst_obj[inst_name]]
     end.to_h
     ext_hash = Dir.glob(@gen_dir / "arch" / "ext" / "**" / "*.yaml").map do |f|
-      ext_obj = YAML.load_file(f)
+      ext_obj = YamlLoader.load(f)
       ext_name = ext_obj.keys[0]
       [ext_name, ext_obj[ext_name]]
     end.to_h
     profile_family_hash = Dir.glob($root / "arch" / "profile_family" / "**" / "*.yaml").map do |f|
-      profile_obj = YAML.load_file(f)
+      profile_obj = YamlLoader.load(f)
       profile_name = profile_obj.keys[0]
       profile_obj[profile_name]["name"] = profile_name
       profile_obj[profile_name]["__source"] = f
       [profile_name, profile_obj[profile_name]]
     end.to_h
     profile_hash = Dir.glob($root / "arch" / "profile" / "**" / "*.yaml").map do |f|
-      profile_obj = YAML.load_file(f)
+      profile_obj = YamlLoader.load(f)
       profile_name = profile_obj.keys[0]
       profile_obj[profile_name]["name"] = profile_name
       profile_obj[profile_name]["__source"] = f
       [profile_name, profile_obj[profile_name]]
     end.to_h
     manual_hash = {}
     Dir.glob($root / "arch" / "manual" / "**" / "contents.yaml").map do |f|
-      manual_version = YAML.load_file(f)
+      manual_version = YamlLoader.load(f)
       manual_id = manual_version["manual"]
       unless manual_hash.key?(manual_id)
         manual_info_files = Dir.glob($root / "arch" / "manual" / "**" / "#{manual_id}.yaml")
         raise "Could not find manual info '#{manual_id}'.yaml, needed by #{f}" if manual_info_files.empty?
         raise "Found multiple manual infos '#{manual_id}'.yaml, needed by #{f}" if manual_info_files.size > 1
 
         manual_info_file = manual_info_files.first
-        manual_hash[manual_id] = YAML.load_file(manual_info_file)
+        manual_hash[manual_id] = YamlLoader.load(manual_info_file)
         manual_hash[manual_id]["__source"] = manual_info_file
         # TODO: schema validation
       end
 
       manual_hash[manual_id]["versions"] ||= []
-      manual_hash[manual_id]["versions"] << YAML.load_file(f)
+      manual_hash[manual_id]["versions"] << YamlLoader.load(f)
       # TODO: schema validation
       manual_hash[manual_id]["versions"].last["__source"] = f
     end
     crd_family_hash = Dir.glob($root / "arch" / "crd_family" / "**" / "*.yaml").map do |f|
-      family_obj = YAML.load_file(f, permitted_classes: [Date])
+      family_obj = YamlLoader.load(f, permitted_classes: [Date])
       family_name = family_obj.keys[0]
       family_obj[family_name]["name"] = family_name
       family_obj[family_name]["__source"] = f
       [family_name, family_obj[family_name]]
     end.to_h
     crd_hash = Dir.glob($root / "arch" / "crd" / "**" / "*.yaml").map do |f|
-      crd_obj = YAML.load_file(f, permitted_classes: [Date])
+      crd_obj = YamlLoader.load(f, permitted_classes: [Date])
       crd_name = crd_obj.keys[0]
       crd_obj[crd_name]["name"] = crd_name
       crd_obj[crd_name]["__source"] = f
@@ -546,99 +545,6 @@ def arch_overlay_path_for(type, name)
   def schema_path_for(type) = Validator::SCHEMA_PATHS[type]
   private :schema_path_for
 
-  # Render a architecture definition file and save it to gen_dir / .rendered_arch
-  #
-  # Will not re-render if rendered file already exists and sources have not changed
-  #
-  # @param type [Symbol] Type of the object (@see Validator::SCHEMA_PATHS)
-  # @param name [#to_s] Name of the object
-  # @param extra_env [Hash,NilObject] Optional hash with extra enviornment variables for the render
-  # @return [Pathname,nil] Path to generated file, or nil if there is no (valid) definition for type,name
-  def gen_rendered_arch_def(type, name, extra_env = {})
-    gen_path = @gen_dir / ".rendered_arch" / type.to_s / "#{name}.yaml"
-
-    source_path = arch_path_for(type, name)
-    return nil if source_path.nil?
-
-    schema_path = schema_path_for(type)
-
-    if gen_path.exist?
-      # this already exists...see if we need to regenerate it
-      dep_mtime = [File.mtime(__FILE__), source_path.mtime, schema_path.mtime].max
-      return gen_path if gen_path.mtime >= dep_mtime # no update needed
-    end
-
-    trace "Rendering architecture file for #{type}:#{name}"
-
-    # render the source template
-    current_env = env.clone
-    extra_env&.each { |k, v| current_env.define_singleton_method(k) { v } }
-    rendered_def = Tilt["erb"].new(source_path, trim: "-").render(current_env)
-
-    # see if the rendering was empty, meaning that the def isn't valid in this config
-    return nil if rendered_def.nil?
-
-    # write the object
-    FileUtils.mkdir_p gen_path.dirname
-    File.write(gen_path, rendered_def)
-
-    # verify
-    begin
-      @validator.validate_str(rendered_def, type:)
-    rescue Validator::SchemaValidationError => e
-      warn "#{type} definition in #{source_path} did not validate"
-      raise e
-    end
-
-    def_obj = YAML.safe_load(rendered_def)
-
-    raise "#{type} name ('#{name}') must match key in defintion ('#{def_obj.keys[0]}')" if name.to_s != def_obj.keys[0]
-
-    # return path to generated file
-    gen_path
-  end
-  private :gen_rendered_arch_def
-
-  # Render a architecture overlay definition file and save it to gen_dir / .rendered_overlay_arch
-  #
-  # Will not re-render if rendered file already exists and sources have not changed
-  #
-  # @param type [Symbol] Type of the object (@see Validator::SCHEMA_PATHS)
-  # @param name [#to_s] Name of the object
-  # @param extra_env [Hash,NilObject] Optional hash with extra enviornment variables for the render
-  # @return [Pathname] Path to generated file
-  def gen_rendered_arch_overlay_def(type, name, extra_env = {})
-    gen_path = @gen_dir / ".rendered_overlay_arch" / type.to_s / "#{name}.yaml"
-
-    source_path = arch_overlay_path_for(type, name)
-    return nil if source_path.nil?
-
-    if gen_path.exist?
-      # this already exists...see if we need to regenerate it
-      dep_mtime = [File.mtime(__FILE__), source_path.mtime].max
-      return gen_path if gen_path.mtime >= dep_mtime # no update needed
-    end
-
-    trace "Rendering overlay file for #{type}:#{name}"
-
-    # render the source template
-    current_env = env.clone
-    extra_env&.each { |k, v| current_env.define_singleton_method(k) { v } }
-    rendered_def = Tilt["erb"].new(source_path, trim: "-").render(current_env)
-
-    def_obj = YAML.safe_load(rendered_def)
-
-    raise "#{type} name (#{name}) must match key in defintion (#{def_obj.keys[0]})" if name.to_s != def_obj.keys[0]
-
-    # write the object
-    FileUtils.mkdir_p gen_path.dirname
-    File.write(gen_path, YAML.dump(def_obj))
-
-    # return path to generated file
-    gen_path
-  end
-  private :gen_rendered_arch_overlay_def
-
   # generate a merged definition from rendered arch and overlay, and write it to gen / .merged_arch
   #
   # Skips if gen file already exists and sources are older
@@ -666,14 +572,16 @@ def gen_merged_def(type, arch_path, overlay_path)
     FileUtils.mkdir_p merged_path.dirname
     if overlay_path.nil?
       # no overlay, just copy arch
-      FileUtils.cp arch_path, merged_path
+      merged_path.write YAML.dump(YamlLoader.load(arch_path))
+      # FileUtils.cp arch_path, merged_path
     elsif arch_path.nil?
       # no arch, overlay is arch
-      FileUtils.cp overlay_path, merged_path
+      merged_path.write YAML.dump(YamlLoader.load(overlay_path))
+      # FileUtils.cp overlay_path, merged_path
     else
       # arch and overlay, do the merge
-      arch_obj = YAML.load_file(arch_path)
-      overlay_obj = YAML.load_file(overlay_path)
+      arch_obj = YamlLoader.load(arch_path)
+      overlay_obj = YamlLoader.load(overlay_path)
 
       merge(arch_obj, overlay_obj)
       merged_path.write YAML.dump(arch_obj)
@@ -696,8 +604,8 @@ def gen_merged_def(type, arch_path, overlay_path)
   # @param csr_name [#to_s] CSR name
   # @param extra_env [Hash] Extra enviornment variables to be used when parsing the CSR definition template
   def maybe_add_csr(csr_name, extra_env = {})
-    arch_path         = arch_path_for(:csr, csr_name) # gen_rendered_arch_def(:csr, csr_name, extra_env)
-    arch_overlay_path = arch_overlay_path_for(:csr, csr_name) # gen_rendered_arch_overlay_def(:csr, csr_name, extra_env)
+    arch_path         = arch_path_for(:csr, csr_name)
+    arch_overlay_path = arch_overlay_path_for(:csr, csr_name)
 
     # return immediately if this CSR isn't defined in this config
     raise "No arch or overlay for sr #{csr_name}" if arch_path.nil? && arch_overlay_path.nil?
@@ -797,8 +705,8 @@ def all_known_exts
   end
 
   def maybe_add_ext(ext_name)
-    arch_path         = arch_path_for(:ext, ext_name) # gen_rendered_arch_def(:ext, ext_name)
-    arch_overlay_path = arch_overlay_path_for(:ext, ext_name) # gen_rendered_arch_overlay_def(:ext, ext_name)
+    arch_path         = arch_path_for(:ext, ext_name)
+    arch_overlay_path = arch_overlay_path_for(:ext, ext_name)
 
     # return immediately if this ext isn't defined
     return if arch_path.nil? && arch_overlay_path.nil?
@@ -884,7 +792,7 @@ def exception_codes
 
     @exception_codes = {}
     Dir.glob(@gen_dir / "arch" / "ext" / "*.yaml") do |ext_path|
-      ext_obj = YAML.load_file(ext_path)
+      ext_obj = YamlLoader.load(ext_path)
       ext_obj = ext_obj[ext_obj.keys[0]]
       if ext_obj.key?("exception_codes")
         ext_obj["exception_codes"].each do |exception_code|
@@ -905,7 +813,7 @@ def interrupt_codes
 
     @interrupt_codes = {}
     Dir.glob(@gen_dir / "arch" / "ext" / "*.yaml") do |ext_path|
-      ext_obj = YAML.load_file(ext_path)
+      ext_obj = YamlLoader.load(ext_path)
       ext_obj = ext_obj[ext_obj.keys[0]]
       if ext_obj.key?("interrupt_codes")
         ext_obj["interrupt_codes"].each do |interrupt_code|
@@ -941,8 +849,8 @@ def possible_xlens
   # @param inst_name [#to_s] instruction name
   # @param extra_env [Hash] Extra options to add into the rendering enviornment
   def maybe_add_inst(inst_name, extra_env = {})
-    arch_path         = arch_path_for(:inst, inst_name) # gen_rendered_arch_def(:inst, inst_name, extra_env)
-    arch_overlay_path = arch_overlay_path_for(:inst, inst_name) # gen_rendered_arch_overlay_def(:inst, inst_name, extra_env)
+    arch_path         = arch_path_for(:inst, inst_name)
+    arch_overlay_path = arch_overlay_path_for(:inst, inst_name)
 
     # return immediately if inst isn't defined in this config
     raise "No arch or overlay for instruction #{inst_name}" if arch_path.nil? && arch_overlay_path.nil?