Skip to content

Commit bc1d84f

Browse files
jviottijviotti
committed
Generate XBRL UTR schemas using jq templates
Signed-off-by: Juan Cruz Viotti <[email protected]>
1 parent d7e0b6b commit bc1d84f

File tree

56 files changed

+1760
-159
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1760
-159
lines changed

‎Makefile‎

Lines changed: 71 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,59 @@ GENERATED = \
3333
schemas/iso/currency/2015/historical/alpha-code.json \
3434
schemas/iso/currency/2015/historical/alpha-currency.json \
3535
schemas/iso/currency/2015/historical/numeric-code.json \
36-
schemas/iso/currency/2015/historical/numeric-currency.json
36+
schemas/iso/currency/2015/historical/numeric-currency.json \
37+
schemas/xbrl/utr/area-item-type-normative.json \
38+
schemas/xbrl/utr/area-item-type.json \
39+
schemas/xbrl/utr/duration-item-type-normative.json \
40+
schemas/xbrl/utr/duration-item-type.json \
41+
schemas/xbrl/utr/electric-charge-item-type-normative.json \
42+
schemas/xbrl/utr/electric-charge-item-type.json \
43+
schemas/xbrl/utr/electric-current-item-type-normative.json \
44+
schemas/xbrl/utr/electric-current-item-type.json \
45+
schemas/xbrl/utr/energy-item-type-normative.json \
46+
schemas/xbrl/utr/energy-item-type.json \
47+
schemas/xbrl/utr/energy-per-monetary-item-type-normative.json \
48+
schemas/xbrl/utr/energy-per-monetary-item-type.json \
49+
schemas/xbrl/utr/flow-item-type-normative.json \
50+
schemas/xbrl/utr/flow-item-type.json \
51+
schemas/xbrl/utr/force-item-type-normative.json \
52+
schemas/xbrl/utr/force-item-type.json \
53+
schemas/xbrl/utr/frequency-item-type-normative.json \
54+
schemas/xbrl/utr/frequency-item-type.json \
55+
schemas/xbrl/utr/ghg-emissions-item-type-normative.json \
56+
schemas/xbrl/utr/ghg-emissions-item-type.json \
57+
schemas/xbrl/utr/ghg-emissions-per-monetary-item-type-normative.json \
58+
schemas/xbrl/utr/ghg-emissions-per-monetary-item-type.json \
59+
schemas/xbrl/utr/length-item-type-normative.json \
60+
schemas/xbrl/utr/length-item-type.json \
61+
schemas/xbrl/utr/mass-item-type-normative.json \
62+
schemas/xbrl/utr/mass-item-type.json \
63+
schemas/xbrl/utr/memory-item-type-normative.json \
64+
schemas/xbrl/utr/memory-item-type.json \
65+
schemas/xbrl/utr/monetary-item-type-normative.json \
66+
schemas/xbrl/utr/monetary-item-type.json \
67+
schemas/xbrl/utr/per-share-item-type-normative.json \
68+
schemas/xbrl/utr/per-share-item-type.json \
69+
schemas/xbrl/utr/per-unit-item-type-normative.json \
70+
schemas/xbrl/utr/per-unit-item-type.json \
71+
schemas/xbrl/utr/plane-angle-item-type-normative.json \
72+
schemas/xbrl/utr/plane-angle-item-type.json \
73+
schemas/xbrl/utr/power-item-type-normative.json \
74+
schemas/xbrl/utr/power-item-type.json \
75+
schemas/xbrl/utr/pressure-item-type-normative.json \
76+
schemas/xbrl/utr/pressure-item-type.json \
77+
schemas/xbrl/utr/pure-item-type-normative.json \
78+
schemas/xbrl/utr/pure-item-type.json \
79+
schemas/xbrl/utr/shares-item-type-normative.json \
80+
schemas/xbrl/utr/shares-item-type.json \
81+
schemas/xbrl/utr/temperature-item-type-normative.json \
82+
schemas/xbrl/utr/temperature-item-type.json \
83+
schemas/xbrl/utr/voltage-item-type-normative.json \
84+
schemas/xbrl/utr/voltage-item-type.json \
85+
schemas/xbrl/utr/volume-item-type-normative.json \
86+
schemas/xbrl/utr/volume-item-type.json \
87+
schemas/xbrl/utr/volume-per-monetary-item-type-normative.json \
88+
schemas/xbrl/utr/volume-per-monetary-item-type.json
3789

3890
# TODO: Make `jsonschema fmt` automatically detect test files
3991
all: common test
@@ -42,19 +94,20 @@ all: common test
4294

4395
.PHONY: common
4496
common: $(GENERATED)
45-
$(JSONSCHEMA) metaschema schemas meta --verbose
46-
$(JSONSCHEMA) lint schemas meta --verbose
47-
$(JSONSCHEMA) validate meta/schemas-root.json --verbose $(SCHEMAS)
48-
$(JSONSCHEMA) validate meta/schemas.json --verbose $(SCHEMAS)
49-
$(JSONSCHEMA) validate meta/test.json --verbose $(TESTS)
97+
$(JSONSCHEMA) metaschema schemas meta
98+
$(JSONSCHEMA) lint schemas meta
99+
$(JSONSCHEMA) validate meta/schemas-root.json $(SCHEMAS)
100+
$(JSONSCHEMA) validate meta/schemas.json $(SCHEMAS)
101+
$(JSONSCHEMA) validate meta/test.json $(TESTS)
50102
$(SHELLCHECK) scripts/*.sh
51-
./scripts/schemas-tests-mirror.sh
103+
./scripts/quality-schemas-tests-mirror.sh
104+
./scripts/quality-templates-xbrl-utr-mirror.sh
52105

53106
# TODO: Make `jsonschema fmt` automatically detect test files
54107
.PHONY: lint
55108
lint: common
56-
$(JSONSCHEMA) fmt schemas meta --verbose --check
57-
$(JSONSCHEMA) fmt test --verbose --default-dialect "https://json-schema.org/draft/2020-12/schema"
109+
$(JSONSCHEMA) fmt schemas meta --check
110+
$(JSONSCHEMA) fmt test --default-dialect "https://json-schema.org/draft/2020-12/schema"
58111

59112
.PHONY: test
60113
test:
@@ -83,14 +136,19 @@ schemas/iso/currency/2015/%.json: build/iso/currency/list-one.json templates/iso
83136
$(JQ) --from-file $(word 2,$^) $< > $@
84137
$(JSONSCHEMA) fmt $@
85138

139+
build/xbrl/utr/%.json: scripts/xml2json.py vendor/data/xbrl/utr/%.xml
140+
$(MKDIRP) $(dir $@)
141+
$(PYTHON) $< $(word 2,$^) $@
142+
schemas/xbrl/utr/%.json: build/xbrl/utr/utr.json templates/xbrl/utr/%.jq
143+
$(MKDIRP) $(dir $@)
144+
$(JQ) --from-file $(word 2,$^) $< > $@
145+
$(JSONSCHEMA) fmt $@
146+
86147
generate-iso-language: generate/iso/language/main.py
87148
$(PYTHON) $<
88149
generate-iso-country: generate/iso/country/main.py
89150
$(PYTHON) $<
90-
build/xbrl/utr/%.json: scripts/xml2json.py vendor/data/xbrl/utr/%.xml
91-
$(PYTHON) $< $(word 2,$^) $@
92-
generate-xbrl-utr: generate/xbrl/utr/main.py build/xbrl/utr/utr.json
93-
$(PYTHON) $<
151+
94152

95153
# TODO: Add a `jsonschema pkg` command instead
96154
.PHONY: dist

‎generate/xbrl/utr/main.py‎

Lines changed: 0 additions & 146 deletions
This file was deleted.

scripts/schemas-tests-mirror.sh renamed to scripts/quality-schemas-tests-mirror.sh

File renamed without changes.

‎scripts/quality-templates-xbrl-utr-mirror.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/sh
2+
3+
set -o errexit
4+
set -o nounset
5+
6+
UTR_DATA="$(pwd)/build/xbrl/utr/utr.json"
7+
TEMPLATES_DIR="$(pwd)/templates/xbrl/utr"
8+
EXIT_CODE=0
9+
10+
# Create temporary directory
11+
TMP="$(mktemp -d)"
12+
# shellcheck disable=SC2329
13+
clean() {
14+
# shellcheck disable=SC2317
15+
rm -rf "$TMP"
16+
}
17+
trap clean EXIT
18+
19+
camel_to_kebab() {
20+
echo "$1" | sed 's/\([A-Z]\)/-\1/g' | tr '[:upper:]' '[:lower:]' | sed 's/^-//'
21+
}
22+
23+
if [ ! -f "$UTR_DATA" ]
24+
then
25+
echo "ERROR: UTR data file '$UTR_DATA' does not exist" >&2
26+
exit 1
27+
fi
28+
29+
# Extract all unique itemTypes from the data and convert to kebab-case
30+
# Store them in the temp file first, then validate
31+
jq -r '.["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"][] | .["{http://www.xbrl.org/2009/utr}itemType"]' "$UTR_DATA" | sort -u > "$TMP/item_types.txt"
32+
33+
while IFS= read -r item_type
34+
do
35+
kebab_name=$(camel_to_kebab "$item_type")
36+
echo "$kebab_name" >> "$TMP/expected_templates.txt"
37+
38+
# Check for regular template
39+
expected_template="$TEMPLATES_DIR/${kebab_name}.jq"
40+
if [ ! -f "$expected_template" ]
41+
then
42+
echo "ERROR: itemType '$item_type' is missing template '$expected_template'" >&2
43+
EXIT_CODE=1
44+
fi
45+
46+
expected_normative="$TEMPLATES_DIR/${kebab_name}-normative.jq"
47+
if [ ! -f "$expected_normative" ]
48+
then
49+
echo "ERROR: itemType '$item_type' is missing normative template '$expected_normative'" >&2
50+
EXIT_CODE=1
51+
fi
52+
done < "$TMP/item_types.txt"
53+
54+
# Check the reverse: for each template, verify it corresponds to a known itemType
55+
find "$TEMPLATES_DIR" -type f -name "*.jq" ! -name "*-normative.jq" > "$TMP/templates.txt"
56+
57+
while IFS= read -r template_file
58+
do
59+
base_name=$(basename "$template_file" .jq)
60+
61+
# Check if this base name is in our expected list
62+
if ! grep -q "^${base_name}$" "$TMP/expected_templates.txt"
63+
then
64+
echo "ERROR: Template '$template_file' does not correspond to any itemType in the data" >&2
65+
EXIT_CODE=1
66+
fi
67+
68+
# Also check that corresponding normative template exists
69+
normative_template="${template_file%.jq}-normative.jq"
70+
if [ ! -f "$normative_template" ]
71+
then
72+
echo "ERROR: Template '$template_file' is missing corresponding normative template '$normative_template'" >&2
73+
EXIT_CODE=1
74+
fi
75+
done < "$TMP/templates.txt"
76+
77+
exit $EXIT_CODE

‎templates/xbrl/utr/area-item-type-normative.jq‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"title": ("XBRL UTR area Item Type units (" + .["{http://www.xbrl.org/2009/utr}utr"]["@attributes"].lastUpdated + ")"),
4+
"description": "Valid units for areaItemType as defined in the XBRL Units Type Registry",
5+
"examples": (
6+
.["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"]
7+
| map(select(.["{http://www.xbrl.org/2009/utr}itemType"] == "areaItemType" and .["{http://www.xbrl.org/2009/utr}status"] == "REC"))
8+
9+
| .[0:3]
10+
| map(.["{http://www.xbrl.org/2009/utr}unitId"])
11+
),
12+
"x-license": "https://github.com/sourcemeta/std/blob/main/LICENSE",
13+
"x-links": [
14+
"https://www.xbrl.org/specification/utr/rec-2013-11-18/utr-rec-2013-11-18-clean.html",
15+
"https://www.xbrl.org/utr/utr.xml"
16+
],
17+
"anyOf": (
18+
.["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"]
19+
| map(select(.["{http://www.xbrl.org/2009/utr}itemType"] == "areaItemType" and .["{http://www.xbrl.org/2009/utr}status"] == "REC"))
20+
21+
| map(
22+
{
23+
"const": .["{http://www.xbrl.org/2009/utr}unitId"],
24+
"title": .["{http://www.xbrl.org/2009/utr}unitName"],
25+
"description": (.["{http://www.xbrl.org/2009/utr}definition"] | gsub("\\s{2,}"; " ") | if endswith(".") then .[:-1] else . end)
26+
} +
27+
(if .["{http://www.xbrl.org/2009/utr}symbol"] then {"x-symbol": .["{http://www.xbrl.org/2009/utr}symbol"]} else {} end) +
28+
(if .["{http://www.xbrl.org/2009/utr}status"] then {"x-status": .["{http://www.xbrl.org/2009/utr}status"]} else {} end)
29+
)
30+
)
31+
}

0 commit comments

Comments
 (0)