Merge pull request #211 from riscv/209-params-json-to-adoc

209 params json to adoc

Author: james-ball-qualcomm

Makefile

@@ -23,6 +23,8 @@ PARAMS_TESTS_DIR := $(TESTS_DIR)/params
 TAGS_TESTS_DIR := $(TESTS_DIR)/tags
 TAG_CHANGES_TESTS_DIR := $(TESTS_DIR)/tag-changes
 ADOC2HTML_TESTS_DIR := $(TESTS_DIR)/adoc2html
+SHARED_UTILS_TESTS_DIR := $(TESTS_DIR)/shared_utils
+TEXT_TO_HTML_TESTS_DIR := $(TESTS_DIR)/text_to_html
 NORM_RULE_DEF_DIR := $(NORM_RULE_TESTS_DIR)
 NORM_RULE_EXPECTED_DIR := $(NORM_RULE_TESTS_DIR)/expected
 PARAMS_DEF_DIR := $(PARAMS_TESTS_DIR)
@@ -36,6 +38,11 @@ DETECT_TAG_CHANGES_TOOL := $(TOOLS_DIR)/detect_tag_changes.py
 DETECT_TAG_CHANGES_PYTHON := python3 $(DETECT_TAG_CHANGES_TOOL)
 CREATE_PARAMS_TOOL := $(TOOLS_DIR)/create_params.py
 CREATE_PARAMS_PYTHON := python3 $(CREATE_PARAMS_TOOL)
+CREATE_PARAM_ADOC_FILES_TOOL := $(TOOLS_DIR)/create_param_appendix.py
+CREATE_PARAM_ADOC_FILES_PYTHON := python3 $(CREATE_PARAM_ADOC_FILES_TOOL)
+PARAM_ADOC_TEST_SCRIPT := $(PARAMS_TESTS_DIR)/test_param_appendix.py
+PARAM_ADOC_TEST_PYTHON := python3 $(PARAM_ADOC_TEST_SCRIPT)
+PARAMS_ADOC_TEMPLATE := $(PARAMS_TESTS_DIR)/test-param-appendix-template.adoc
 
 # Stuff for building test standards document in HTML to have links into it.
 DOCS = test-ch1 test-ch2
@@ -78,6 +85,9 @@ BUILT_NORM_RULES_HTML := $(BUILD_DIR)/$(NORM_RULE_HTML_OUTPUT_FNAME)
 BUILT_NORM_RULES_TAGS_NO_RULES := $(BUILD_DIR)/$(NORM_RULE_TAGS_NO_RULES_OUTPUT_FNAME)
 BUILT_PARAMS_JSON := $(BUILD_DIR)/$(PARAMS_JSON_OUTPUT_FNAME)
 BUILT_PARAMS_HTML := $(BUILD_DIR)/$(PARAMS_HTML_OUTPUT_FNAME)
+BUILT_PARAM_ADOC_DIR := $(BUILD_DIR)/test-param-appendix-adoc-includes
+BUILT_PARAM_ADOC_STAMP := $(BUILD_DIR)/test-param-adoc.done
+BUILT_PARAMS_ADOC := $(BUILD_DIR)/test-param-appendix.adoc
 
 # Combine separate fnames into lists.
 BUILT_TEST_HTML_FNAMES := $(BUILT_TEST_CH1_HTML_FNAME) $(BUILT_TEST_CH2_HTML_FNAME)
@@ -91,12 +101,18 @@ EXPECTED_NORM_RULES_JSON := $(NORM_RULE_EXPECTED_DIR)/$(NORM_RULE_JSON_OUTPUT_FN
 EXPECTED_NORM_RULES_HTML := $(NORM_RULE_EXPECTED_DIR)/$(NORM_RULE_HTML_OUTPUT_FNAME)
 EXPECTED_PARAMS_JSON := $(PARAMS_EXPECTED_DIR)/$(PARAMS_JSON_OUTPUT_FNAME)
 EXPECTED_PARAMS_HTML := $(PARAMS_EXPECTED_DIR)/$(PARAMS_HTML_OUTPUT_FNAME)
+EXPECTED_PARAMS_ADOC := $(PARAMS_EXPECTED_DIR)/test-param-appendix.adoc
+EXPECTED_PARAM_ADOC_DIR := $(PARAMS_EXPECTED_DIR)/test-param-appendix-adoc-includes
 
 # Normative rule definition input YAML files.
 GOOD_NORM_RULE_DEF_FILES := $(NORM_RULE_DEF_DIR)/test-ch1.yaml $(NORM_RULE_DEF_DIR)/test-ch2.yaml
 BAD_NORM_RULE_DEF_FILES := $(NORM_RULE_DEF_DIR)/missing_tag_refs.yaml
 PARAM_DEF_TEST_FILES := $(PARAMS_TESTS_DIR)/test-ch1.yaml $(PARAMS_TESTS_DIR)/test-ch2.yaml
 ADOC2HTML_TEST_SCRIPT := $(ADOC2HTML_TESTS_DIR)/test_adoc_to_html.py
+SHARED_UTILS_TEST_SCRIPT := $(SHARED_UTILS_TESTS_DIR)/test_shared_utils.py
+ADOC_TO_HTML_UNIT_TEST_SCRIPT := $(TEXT_TO_HTML_TESTS_DIR)/test_adoc_to_html_unit.py
+DEF_TEXT_TO_HTML_UNIT_TEST_SCRIPT := $(TEXT_TO_HTML_TESTS_DIR)/test_def_text_to_html_unit.py
+TAG_TEXT_TO_HTML_UNIT_TEST_SCRIPT := $(TEXT_TO_HTML_TESTS_DIR)/test_tag_text_to_html_unit.py
 
 # Add -t to each normative tag input filename and add prefix of "/" to make into absolute pathname.
 NORM_TAG_FILE_ARGS := $(foreach relative_pname,$(BUILT_TEST_NORM_TAGS_FNAMES),-t /$(relative_pname))
@@ -174,21 +190,22 @@ all: test
 
 # Build tests and compare against expected
 .PHONY: test
-test: build-tests compare-tests test-tag-changes test-adoc2html
+test: build-tests compare-tests test-tag-changes test-adoc2html test-shared-utils test-text-to-html
 
 # Build tests
-.PHONY: build-tests build-test-tags build-test-norm-rules-json build-test-norm-rules-html build-test-tags-without-rules build-test-params-json build-test-params-html
-build-tests: build-test-tags build-test-norm-rules-json build-test-norm-rules-html build-test-tags-without-rules build-test-params-json build-test-params-html
+.PHONY: build-tests build-test-tags build-test-norm-rules-json build-test-norm-rules-html build-test-tags-without-rules build-test-params-json build-test-params-html build-test-param-adoc
+build-tests: build-test-tags build-test-norm-rules-json build-test-norm-rules-html build-test-tags-without-rules build-test-params-json build-test-params-html build-test-param-adoc
 build-test-tags: $(BUILT_TEST_NORM_TAGS_FNAMES) $(BUILT_DUPLICATE_NORM_TAGS_FNAME)
 build-test-norm-rules-json: $(BUILT_NORM_RULES_JSON)
 build-test-norm-rules-html: $(BUILT_NORM_RULES_HTML)
 build-test-tags-without-rules: $(BUILT_NORM_RULES_TAGS_NO_RULES)
 build-test-params-json: $(BUILT_PARAMS_JSON)
 build-test-params-html: $(BUILT_PARAMS_HTML)
+build-test-param-adoc: $(BUILT_PARAM_ADOC_STAMP)
 
 # Compare tests against expected
 .PHONY: compare-tests
-compare-tests: compare-test-tags compare-test-norm-rules-json compare-test-norm-rules-html compare-test-params-json compare-test-params-html
+compare-tests: compare-test-tags compare-test-norm-rules-json compare-test-norm-rules-html compare-test-params-json compare-test-params-html compare-test-params-adoc compare-test-param-adoc-files
 
 .PHONY: compare-test-tags
 compare-test-tags: compare-test-ch1-tags compare-test-ch2-tags
@@ -223,6 +240,16 @@ compare-test-params-html: $(EXPECTED_PARAMS_HTML) $(BUILT_PARAMS_HTML)
 	@echo "CHECKING HTML BUILT PARAMS AGAINST EXPECTED PARAMS"
 	diff $(EXPECTED_PARAMS_HTML) $(BUILT_PARAMS_HTML) && echo "diff PASSED" || (echo "diff FAILED"; exit 1)
 
+.PHONY: compare-test-params-adoc
+compare-test-params-adoc: $(EXPECTED_PARAMS_ADOC) $(BUILT_PARAMS_ADOC)
+	@echo "CHECKING PARAMS ADOC AGAINST EXPECTED"
+	diff $(EXPECTED_PARAMS_ADOC) $(BUILT_PARAMS_ADOC) && echo "diff PASSED" || (echo "diff FAILED"; exit 1)
+
+.PHONY: compare-test-param-adoc-files
+compare-test-param-adoc-files: $(EXPECTED_PARAM_ADOC_DIR) $(BUILT_PARAM_ADOC_DIR)
+	@echo "CHECKING GENERATED PARAM ADOC FILES AGAINST EXPECTED"
+	diff -r $(EXPECTED_PARAM_ADOC_DIR) $(BUILT_PARAM_ADOC_DIR) && echo "diff PASSED" || (echo "diff FAILED"; exit 1)
+
 # Test tag change detection
 .PHONY: test-tag-changes test-tag-changes-basic test-tag-changes-verbose test-tag-changes-no-changes test-tag-changes-additions-only test-tag-changes-whitespace-only test-tag-changes-formatting-only test-tag-changes-update
 test-tag-changes: test-tag-changes-basic test-tag-changes-verbose test-tag-changes-no-changes test-tag-changes-additions-only test-tag-changes-whitespace-only test-tag-changes-formatting-only test-tag-changes-update
@@ -264,9 +291,34 @@ test-adoc2html: $(ADOC2HTML_TEST_SCRIPT) $(TOOLS_DIR)/adoc_to_html.py
 	@echo "TESTING ADOC2HTML CONVERSION"
 	python3 $(ADOC2HTML_TEST_SCRIPT) && echo "test-adoc2html PASSED" || (echo "test-adoc2html FAILED"; exit 1)
 
+# Test shared_utils helpers in isolation.
+.PHONY: test-shared-utils
+test-shared-utils: $(SHARED_UTILS_TEST_SCRIPT) $(TOOLS_DIR)/shared_utils.py
+	@echo "TESTING SHARED_UTILS HELPERS"
+	python3 $(SHARED_UTILS_TEST_SCRIPT) && echo "test-shared-utils PASSED" || (echo "test-shared-utils FAILED"; exit 1)
+
+# Test converter helpers in isolation.
+.PHONY: test-text-to-html
+test-text-to-html: test-adoc-to-html-unit test-def-text-to-html-unit test-tag-text-to-html-unit
+
+.PHONY: test-adoc-to-html-unit
+test-adoc-to-html-unit: $(ADOC_TO_HTML_UNIT_TEST_SCRIPT) $(TOOLS_DIR)/adoc_to_html.py
+	@echo "TESTING ADOC_TO_HTML UNIT HELPERS"
+	python3 $(ADOC_TO_HTML_UNIT_TEST_SCRIPT) && echo "test-adoc-to-html-unit PASSED" || (echo "test-adoc-to-html-unit FAILED"; exit 1)
+
+.PHONY: test-def-text-to-html-unit
+test-def-text-to-html-unit: $(DEF_TEXT_TO_HTML_UNIT_TEST_SCRIPT) $(TOOLS_DIR)/def_text_to_html.py $(TOOLS_DIR)/adoc_to_html.py
+	@echo "TESTING DEF_TEXT_TO_HTML UNIT HELPERS"
+	python3 $(DEF_TEXT_TO_HTML_UNIT_TEST_SCRIPT) && echo "test-def-text-to-html-unit PASSED" || (echo "test-def-text-to-html-unit FAILED"; exit 1)
+
+.PHONY: test-tag-text-to-html-unit
+test-tag-text-to-html-unit: $(TAG_TEXT_TO_HTML_UNIT_TEST_SCRIPT) $(TOOLS_DIR)/tag_text_to_html.py $(TOOLS_DIR)/def_text_to_html.py $(TOOLS_DIR)/adoc_to_html.py
+	@echo "TESTING TAG_TEXT_TO_HTML UNIT HELPERS"
+	python3 $(TAG_TEXT_TO_HTML_UNIT_TEST_SCRIPT) && echo "test-tag-text-to-html-unit PASSED" || (echo "test-tag-text-to-html-unit FAILED"; exit 1)
+
 # Update expected files from built files
 .PHONY: update-expected
-update-expected: update-test-tags update-test-norm-rules-json update-test-norm-rules-html update-test-params-json update-test-params-html
+update-expected: update-test-tags update-test-norm-rules-json update-test-norm-rules-html update-test-params-json update-test-params-html update-test-params-adoc update-test-param-adoc-files
 
 .PHONY: update-test-tags
 update-test-tags: update-test-ch1-tags update-test-ch2-tags
@@ -297,6 +349,17 @@ update-test-params-html: $(BUILT_PARAMS_HTML)
 	mkdir -p $(PARAMS_EXPECTED_DIR)
 	cp -f $(BUILT_PARAMS_HTML) $(EXPECTED_PARAMS_HTML)
 
+.PHONY: update-test-params-adoc
+update-test-params-adoc: $(BUILT_PARAMS_ADOC)
+	mkdir -p $(PARAMS_EXPECTED_DIR)
+	cp -f $(BUILT_PARAMS_ADOC) $(EXPECTED_PARAMS_ADOC)
+
+.PHONY: update-test-param-adoc-files
+update-test-param-adoc-files: $(BUILT_PARAM_ADOC_DIR)
+	rm -rf $(EXPECTED_PARAM_ADOC_DIR)
+	mkdir -p $(EXPECTED_PARAM_ADOC_DIR)
+	cp -f $(BUILT_PARAM_ADOC_DIR)/*.adoc $(EXPECTED_PARAM_ADOC_DIR)
+
 # Build normative tags with ch1 adoc input
 $(BUILT_TEST_CH1_NORM_TAGS_FNAME): $(NORM_RULE_TESTS_DIR)/$(TEST_CH1_INPUT_ADOC_FNAME) $(CONVERTERS_DIR)/$(TAGS_BACKEND)
 	$(WORKDIR_SETUP)
@@ -337,6 +400,17 @@ $(BUILT_PARAMS_HTML): $(BUILT_NORM_RULES_JSON) $(PARAM_DEF_TEST_FILES) $(CREATE_
 	$(DOCKER_CMD) $(DOCKER_QUOTE) $(CREATE_PARAMS_PYTHON) --html --norm-rules $(BUILT_NORM_RULES_JSON) $(PARAM_DEF_TEST_ARGS) --output $(BUILT_PARAMS_HTML) $(DOCKER_QUOTE)
 	$(WORKDIR_TEARDOWN)
 
+# Build parameter AsciiDoc row fragments from generated params JSON.
+$(BUILT_PARAM_ADOC_STAMP): $(BUILD_DIR)/$(PARAMS_JSON_OUTPUT_FNAME) $(CREATE_PARAM_ADOC_FILES_TOOL) $(PARAM_ADOC_TEST_SCRIPT) $(PARAMS_ADOC_TEMPLATE)
+	$(WORKDIR_SETUP)
+	cp -f $(BUILD_DIR)/$(PARAMS_JSON_OUTPUT_FNAME) $@.workdir/$(BUILD_DIR)
+	$(DOCKER_CMD) $(DOCKER_QUOTE) $(CREATE_PARAM_ADOC_FILES_PYTHON) --input $(BUILD_DIR)/$(PARAMS_JSON_OUTPUT_FNAME) --output-dir $(BUILT_PARAM_ADOC_DIR) && touch $(BUILT_PARAM_ADOC_STAMP) $(DOCKER_QUOTE)
+	$(DOCKER_CMD) $(DOCKER_QUOTE) $(PARAM_ADOC_TEST_PYTHON) --template $(PARAMS_ADOC_TEMPLATE) --adoc-dir $(BUILT_PARAM_ADOC_DIR) --output $(BUILT_PARAMS_ADOC) $(DOCKER_QUOTE)
+	rm -rf $(BUILT_PARAM_ADOC_DIR)
+	mv $@.workdir/$(BUILT_PARAM_ADOC_DIR) $(BUILT_PARAM_ADOC_DIR)
+	mv $@.workdir/$(BUILT_PARAMS_ADOC) $(BUILT_PARAMS_ADOC)
+	$(WORKDIR_TEARDOWN)
+
 # Build normative rules with HTML output format
 $(BUILT_NORM_RULES_HTML): $(BUILT_TEST_NORM_TAGS_FNAMES) $(GOOD_NORM_RULE_DEF_FILES) $(BUILT_TEST_HTML_FNAMES)
 	$(WORKDIR_SETUP)

tests/norm-rule/expected/test-ch1-norm-tags.json

@@ -64,7 +64,8 @@
     "norm:int64": "This is a 64-bit signed integer.",
     "norm:uint-1to100": "This is an unsigned integer in the range of 1 to 100.",
     "norm:list-str": "Name|Description\n===\nABC|Letters \"ABC\"¶DEF|Letters \"DEF\"¶GHI|Letters \"GHI\"\n===",
-    "norm:list-int": "This is a list with values 1, 2 and 4."
+    "norm:list-int": "This is a list with values 1, 2 and 4.",
+    "norm:foobar-ch1": "This is the Ch1 part of the foobar normative rule."
   },
   "sections": {
     "title": "",
@@ -195,7 +196,8 @@
           "norm:int64",
           "norm:uint-1to100",
           "norm:list-str",
-          "norm:list-int"
+          "norm:list-int",
+          "norm:foobar-ch1"
         ]
       }
     ],

tests/norm-rule/expected/test-ch2-norm-tags.json

@@ -10,6 +10,13 @@
     "norm:ROSE_COLORS": "You can use red or yellow roses. The tags backend removes some adoc formating so I don't see bold text but I can see ^superscript^ as long as I use the inline anchor and not the paragraph anchor.",
     "norm:XLEN": "We use the term XLEN to refer to the width of an integer register in bits (either 32 or 64). No superscript in paragraph anchors for some reason.",
     "norm:ASIDLEN": "The number of ASID bits is UNSPECIFIED and may be zero. The number of implemented ASID bits, termed ASIDLEN, may be determined by writing one to every bit position in the ASID field, then reading back the value in satp to see which bit positions in the ASID field hold a one.",
+    "norm:foobar-ch2": "This is the Ch2 part of the foobar normative rule.",
+    "norm:mock-inst-x": "Here’s a normative rule for mock instruction \"X\".",
+    "norm:base-rv32i": "Here’s a normative rule for the RV32I base ISA.",
+    "norm:base-rv32i-rv64i": "Here’s a normative rule for the RV32I or RV64I base ISA.",
+    "norm:mock-csr-x": "Here’s a normative rule for mock CSR \"X\".",
+    "norm:mock-csr-field-y": "Here’s a normative rule for mock CSR field \"Y\".",
+    "norm:mock-ext-dep-A-on-B": "Here’s a normative rule for mock extension dependency of extension A on extension B.",
     "norm:csr_field_warl1": "This field is write-any-read-legal.",
     "norm:csr_field_wlrl": "This field is write-legal-read-legal.",
     "norm:csr_field_warl2": "This field is also write-any-read-legal."
@@ -37,13 +44,20 @@
             ]
           },
           {
-            "title": "Chapter 2.2 - Implementation-Defined Rules",
-            "id": "_chapter_2_2_implementation_defined_rules",
+            "title": "Chapter 2.2 - Implementation-Defined Behaviors",
+            "id": "_chapter_2_2_implementation_defined_behaviors",
             "children": [],
             "tags": [
               "norm:ROSE_COLORS",
               "norm:XLEN",
-              "norm:ASIDLEN"
+              "norm:ASIDLEN",
+              "norm:foobar-ch2",
+              "norm:mock-inst-x",
+              "norm:base-rv32i",
+              "norm:base-rv32i-rv64i",
+              "norm:mock-csr-x",
+              "norm:mock-csr-field-y",
+              "norm:mock-ext-dep-A-on-B"
             ]
           },
           {