Skip to content
This repository was archived by the owner on Feb 25, 2020. It is now read-only.

Commit 2bcd455

Browse files
aparcaraparcar
committed
Backport JSON info files patch
While unmerged, patch it here openwrt/openwrt#2192 Signed-off-by: Paul Spooren <mail@aparcar.org>
1 parent 53fda31 commit 2bcd455

File tree

4 files changed

+393
-0
lines changed

4 files changed

+393
-0
lines changed

‎meta‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,3 +204,7 @@ make "$1" \
204204
EXTRA_IMAGE_NAME="$EXTRA_IMAGE_NAME" \
205205
FILES="$FILES"
206206
)
207+
208+
[ "$1" = "image" ] && (cd "$IB_DIR" &&
209+
make "jsonmergeimageinfo" OUTPUT_DIR="$ROOT_DIR/bin/$DISTRO/$VERSION"
210+
)

‎patches/0001-build-create-JSON-files-containing-image-info.patch‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
From f9777b0434a1854e723026347b3e06b19d8a456b Mon Sep 17 00:00:00 2001
2+
From: Paul Spooren <mail@aparcar.org>
3+
Date: Sun, 18 Aug 2019 09:56:45 -1000
4+
Subject: [PATCH 1/2] build: create JSON files containing image info
5+
6+
The JSON info files contain details about the created firmware images
7+
per device and are stored next to the created images.
8+
9+
The JSON files are stored as "$(IMAGE_PREFIX).json" and contain some
10+
device/image meta data as well as a list of created firmware images.
11+
12+
An example of openwrt-ath79-generic-tplink_tl-wdr3600-v1.json
13+
14+
{
15+
"id": "tplink_tl-wdr3600-v1",
16+
"image_prefix": "openwrt-ath79-generic-tplink_tl-wdr3600-v1",
17+
"image_size": "7936k",
18+
"images": [
19+
{
20+
"name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-sysupgrade.bin",
21+
"sha256": "60ef977447d57ffe406f1f6170860be8043654d961933b73645850b25c6a1990",
22+
"type": "sysupgrade"
23+
},
24+
{
25+
"name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-factory.bin",
26+
"sha256": "c6fae436b13f512e65ef05c0ae94308dd1cc9e20fd929dd3e0422574fe58d2b5",
27+
"type": "factory"
28+
}
29+
],
30+
"metadata_version": 1,
31+
"model": "TL-WDR3600",
32+
"supported_devices": [
33+
"tplink,tl-wdr3600-v1",
34+
"tl-wdr4300"
35+
],
36+
"target": "ath79/generic",
37+
"title": [
38+
"TP-Link TL-WDR3600 v1"
39+
],
40+
"variant": "v1",
41+
"vendor": "TP-Link",
42+
"version_commit": "r10764-84c103509a",
43+
"version_number": "SNAPSHOT"
44+
}
45+
46+
Signed-off-by: Paul Spooren <mail@aparcar.org>
47+
---
48+
config/Config-build.in | 7 +++++
49+
include/image.mk | 25 ++++++++++++++++-
50+
scripts/json_add_image_info.py | 51 ++++++++++++++++++++++++++++++++++
51+
3 files changed, 82 insertions(+), 1 deletion(-)
52+
create mode 100755 scripts/json_add_image_info.py
53+
54+
diff --git a/include/image.mk b/include/image.mk
55+
index c6a6ab7993..cfb2e2a90a 100644
56+
--- a/include/image.mk
57+
+++ b/include/image.mk
58+
@@ -554,7 +554,28 @@ define Device/Build/image
59+
60+
$(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2))
61+
cp $$^ $$@
62+
-
63+
+ $(if $(CONFIG_JSON_ADD_IMAGE_INFO), \
64+
+ DEVICE_ID="$(DEVICE_NAME)" \
65+
+ TOPDIR="$(TOPDIR)" \
66+
+ BIN_DIR="$(BIN_DIR)" \
67+
+ IMAGE_NAME="$(IMAGE_NAME)" \
68+
+ IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
69+
+ IMAGE_SIZE="$(IMAGE_SIZE)" \
70+
+ IMAGE_PREFIX="$(IMAGE_PREFIX)" \
71+
+ DEVICE_TITLE="$(DEVICE_TITLE)" \
72+
+ DEVICE_VENDOR="$(DEVICE_VENDOR)" \
73+
+ DEVICE_MODEL="$(DEVICE_MODEL)" \
74+
+ DEVICE_VARIANT="$(DEVICE_VARIANT)" \
75+
+ DEVICE_ALT0_TITLE="$(DEVICE_ALT0_TITLE)" \
76+
+ DEVICE_ALT1_TITLE="$(DEVICE_ALT1_TITLE)" \
77+
+ DEVICE_ALT2_TITLE="$(DEVICE_ALT2_TITLE)" \
78+
+ TARGET="$(BOARD)" \
79+
+ SUBTARGET="$(SUBTARGET)" \
80+
+ VERSION_NUMBER="$(VERSION_NUMBER)" \
81+
+ VERSION_CODE="$(VERSION_CODE)" \
82+
+ SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
83+
+ $(TOPDIR)/scripts/json_add_image_info.py \
84+
+ )
85+
endef
86+
87+
define Device/Build/artifact
88+
@@ -572,6 +593,8 @@ define Device/Build/artifact
89+
endef
90+
91+
define Device/Build
92+
+ $(if $(CONFIG_JSON_ADD_IMAGE_INFO), $(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json))
93+
+
94+
$(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1)))
95+
$(call Device/Build/kernel,$(1))
96+
97+
diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py
98+
new file mode 100755
99+
index 0000000000..31b8d1c123
100+
--- /dev/null
101+
+++ b/scripts/json_add_image_info.py
102+
@@ -0,0 +1,51 @@
103+
+#!/usr/bin/env python3
104+
+
105+
+import json
106+
+import os
107+
+import hashlib
108+
+
109+
+
110+
+def e(variable):
111+
+ return os.environ.get(variable)
112+
+
113+
+
114+
+json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX"))
115+
+
116+
+with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file:
117+
+ image_hash = hashlib.sha256(image_file.read()).hexdigest()
118+
+
119+
+if not os.path.exists(json_path):
120+
+ device_info = {
121+
+ "id": e("DEVICE_ID"),
122+
+ "image_prefix": e("IMAGE_PREFIX"),
123+
+ "image_size": e("IMAGE_SIZE"),
124+
+ "images": [],
125+
+ "metadata_version": 1,
126+
+ "model": e("DEVICE_MODEL"),
127+
+ "supported_devices": e("SUPPORTED_DEVICES").split(),
128+
+ "target": "{}/{}".format(e("TARGET"), e("SUBTARGET")),
129+
+ "title": list(
130+
+ filter(
131+
+ None,
132+
+ [
133+
+ e("DEVICE_TITLE"),
134+
+ e("DEVICE_ALT0_TITLE"),
135+
+ e("DEVICE_ALT1_TITLE"),
136+
+ e("DEVICE_ALT2_TITLE"),
137+
+ ],
138+
+ )
139+
+ ),
140+
+ "variant": e("DEVICE_VARIANT"),
141+
+ "vendor": e("DEVICE_VENDOR"),
142+
+ "version_commit": e("VERSION_CODE"),
143+
+ "version_number": e("VERSION_NUMBER"),
144+
+ }
145+
+else:
146+
+ with open(json_path, "r") as json_file:
147+
+ device_info = json.load(json_file)
148+
+
149+
+image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash}
150+
+device_info["images"].append(image_info)
151+
+
152+
+with open(json_path, "w") as json_file:
153+
+ json.dump(device_info, json_file, sort_keys=True, indent=" ")
154+
--
155+
2.20.1
156+

‎patches/0002-build-add-JSON-info-merge-script.patch‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
From 1ea12b6d1bb067d255bec10eec6be18c73c24786 Mon Sep 17 00:00:00 2001
2+
From: Paul Spooren <mail@aparcar.org>
3+
Date: Sun, 18 Aug 2019 10:47:24 -1000
4+
Subject: [PATCH 2/2] build: add JSON info merge script
5+
6+
Script creates three different files types with different functions:
7+
8+
* map.json
9+
10+
Stored in each target/subtarget folder containing a mapping between
11+
$(SUPPORTED_DEVICES) and JSON image info files. With this file a router
12+
can automatically obtain a sysuprade firmware image.
13+
14+
{
15+
"devices": {
16+
"netgear,r7800": {
17+
"info": "openwrt-ipq806x-generic-netgear_r7800.json",
18+
"sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a"
19+
},
20+
"r7800": {
21+
"info": "openwrt-ipq806x-generic-netgear_r7800.json",
22+
"sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a"
23+
}
24+
},
25+
"metadata_version": 1,
26+
"target": "ipq806x/generic"
27+
}
28+
29+
This mapping style allows device identifiers to change between releases, as it
30+
happend between ar71xx and ath79.
31+
32+
Sha256sums are stored to verify file integrity, later more on that.
33+
34+
This files is stored in $(OUTPUT_DIR) and contains a list of all
35+
targets where images exist for.
36+
37+
* targets.json
38+
39+
{
40+
"metadata_version": 1,
41+
"targets": {
42+
"ar71xx/generic": {
43+
"path": "targets/ath79/generic/map.json",
44+
"sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7"
45+
},
46+
"ath79/generic": {
47+
"path": "targets/ath79/generic/map.json",
48+
"sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7"
49+
},
50+
"ipq806x/generic": {
51+
"path": "targets/ipq806x/generic/map.json",
52+
"sha256": "6afba149bc0b0f22e19ebb49e10d7e2a98e80286f84eafcd13b65384f71f401f"
53+
},
54+
"x86/64": {
55+
"path": "targets/x86/64/map.json",
56+
"sha256": "0f445ca9807d7f21624fff802f33320b30247eedfca636237317a144729c02e1"
57+
}
58+
}
59+
}
60+
61+
This mapping style allows device targets to change between releases, as it
62+
happend between ar71xx and ath79. Devices searching for ar71xx are forwarded to
63+
compatible deivces of ath79.
64+
65+
The workflow would be the following:
66+
67+
* Devices request a (tbd) signed versions.json file from the server
68+
* Parse the file on device and optain the folder containing targets.json
69+
* Request targets.json and optain path to target specific map.json
70+
* Request map.json and optain device specific info file
71+
* Parse info file images for a sysupgrade
72+
* Request sysupgrade and flash device
73+
74+
The versions.json would be signed containing sha256sums of targets.json files.
75+
This allows a chain of trust:
76+
77+
versions.json -> targes.json -> map.json -> device_id.json -> sysupgrade.bin
78+
79+
The versions.json file could look like the following:
80+
81+
{
82+
"versions": [
83+
{
84+
"name": "19.07-SNAPSHOT",
85+
"path": "releases/19.07-SNAPSHOTS",
86+
"sha256": "e659ccba1cd70124138726efabd9dd7a60ee220bf8471a91082c7e05488cac19"
87+
},
88+
{
89+
"name": "Snapshot",
90+
"path": "snapshots",
91+
"sha256": "7c84e6140f35c70d6fe71cd1ca8e5d98d9f88a887a4550adf137d4fd2a1f0ea6"
92+
}
93+
],
94+
"metadata_version": 1
95+
}
96+
97+
* overview.json
98+
99+
Contains a mapping between device title(s) and device info files. Having
100+
this file greatly simpifies firmware image retreiveal. A (web) client can
101+
search trhough the mapping and show the results to an end user.
102+
103+
{
104+
"devices": {
105+
" TL-WR840N v5": "targets/ramips/mt76x8/openwrt-ramips-mt76x8-tplink_tl-wr840n-v5.json",
106+
"7Links PX-4885 4M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-4m.json",
107+
"7Links PX-4885 8M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-8m.json",
108+
"8devices Carambola": "targets/ramips/rt305x/openwrt-ramips-rt305x-8devices_carambola.json",
109+
"8devices Carambola2": "targets/ath79/generic/openwrt-ath79-generic-8dev_carambola2.json",
110+
...
111+
},
112+
"metadata_version": 1
113+
}
114+
115+
Signed-off-by: Paul Spooren <mail@aparcar.org>
116+
---
117+
Makefile | 5 ++
118+
config/Config-build.in | 12 +++++
119+
scripts/json_merge_image_info.py | 75 ++++++++++++++++++++++++++++++
120+
target/imagebuilder/files/Makefile | 4 ++
121+
4 files changed, 96 insertions(+)
122+
create mode 100755 scripts/json_merge_image_info.py
123+
124+
diff --git a/scripts/json_merge_image_info.py b/scripts/json_merge_image_info.py
125+
new file mode 100755
126+
index 0000000000..bc61424dd6
127+
--- /dev/null
128+
+++ b/scripts/json_merge_image_info.py
129+
@@ -0,0 +1,75 @@
130+
+#!/usr/bin/env python3
131+
+
132+
+import json
133+
+import glob
134+
+import os
135+
+import hashlib
136+
+
137+
+output_dir = os.path.join(os.environ.get("OUTPUT_DIR", "./bin"))
138+
+
139+
+dev_overview = {"metadata_version": 1, "devices": {}}
140+
+targets = {"metadata_version": 1, "targets": {}}
141+
+
142+
+# find all json files in ./bin/targets and create sha256 checksums
143+
+targets_dir = os.path.join(output_dir, "targets")
144+
+
145+
+for root, dirs, files in os.walk(targets_dir):
146+
+ current_dir = root[len(targets_dir) + 1 :]
147+
+ # check if root contains one slash aka target/subtarget
148+
+ if current_dir.count("/") == 1:
149+
+ targets["targets"][current_dir] = {
150+
+ "path": "targets/{}/map.json".format(current_dir)
151+
+ }
152+
+ # initialize maps.json for target/subtarget
153+
+ dev_map = {"metadata_version": 1, "target": current_dir, "devices": {}}
154+
+ for file in files:
155+
+ # ignore existing map.json files
156+
+ if file.endswith(".json") and file != "map.json":
157+
+ # load device info
158+
+ with open(os.path.join(root, file), "r") as dev_file:
159+
+ dev_info = json.load(dev_file)
160+
+
161+
+ # generate sha256sum
162+
+ with open(os.path.join(root, file), "rb") as dev_file_b:
163+
+ dev_sha256 = hashlib.sha256(dev_file_b.read()).hexdigest()
164+
+
165+
+ # generate map entry for each supported device
166+
+ for supported in dev_info["supported_devices"]:
167+
+ dev_map["devices"][supported] = {}
168+
+ dev_map["devices"][supported]["info"] = file
169+
+ dev_map["devices"][supported]["sha256"] = dev_sha256
170+
+
171+
+ # path from overview.json to device info files
172+
+ dev_path = "{}/{}".format(dev_info["target"], file)
173+
+
174+
+ # add title(s) to overview
175+
+ for title in dev_info.get("title"):
176+
+ if title in dev_overview["devices"]:
177+
+ print(
178+
+ "WARNING: '{}' pointing to '{}' already exists in overview and will be overwritten with '{}/{}'".format(
179+
+ title,
180+
+ dev_overview["devices"][title],
181+
+ dev_info["target"],
182+
+ file,
183+
+ )
184+
+ )
185+
+ dev_overview["devices"][title] = dev_path
186+
+
187+
+ # write map.json to target/subtarget
188+
+ with open(os.path.join(root, "map.json"), "w") as dev_map_file:
189+
+ json.dump(dev_map, dev_map_file, sort_keys=True, indent=" ")
190+
+
191+
+for target, data in targets["targets"].items():
192+
+ # generate sha256sum of target/subtarget/maps.json
193+
+ with open(os.path.join(output_dir, data["path"]), "rb") as dev_file_b:
194+
+ targets["targets"][target]["sha256"] = hashlib.sha256(
195+
+ dev_file_b.read()
196+
+ ).hexdigest()
197+
+
198+
+# write targets.json to ./bin/
199+
+with open(os.path.join(output_dir, "targets.json"), "w") as targets_file:
200+
+ json.dump(targets, targets_file, sort_keys=True, indent=" ")
201+
+
202+
+# write overview.json to ./bin/
203+
+with open(os.path.join(output_dir, "overview.json"), "w") as dev_overview_file:
204+
+ json.dump(dev_overview, dev_overview_file, sort_keys=True, indent=" ")
205+
diff --git a/Makefile b/Makefile
206+
index 22b2731358..018abc9a23 100644
207+
--- a/Makefile
208+
+++ b/Makefile
209+
@@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build
210+
$(MAKE) -s prepare_rootfs
211+
$(MAKE) -s build_image
212+
$(MAKE) -s checksum
213+
+ $(if $(CONFIG_JSON_MERGE_IMAGE_INFO),$(MAKE) -s jsonmergeimageinfo)
214+
215+
_call_manifest: FORCE
216+
rm -rf $(TARGET_DIR)
217+
@@ -201,6 +202,9 @@ image:
218+
$(if $(BIN_DIR),BIN_DIR="$(BIN_DIR)") \
219+
$(if $(DISABLED_SERVICES),DISABLED_SERVICES="$(DISABLED_SERVICES)"))
220+
221+
+jsonmergeimageinfo: FORCE
222+
+ $(SCRIPT_DIR)/json_merge_image_info.py
223+
+
224+
manifest: FORCE
225+
$(MAKE) -s _check_profile
226+
(unset PROFILE FILES PACKAGES MAKEFLAGS; \
227+
--
228+
2.20.1
229+

‎scripts/activate-json-info-files.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
chmod +x ./scripts/json_*.py
4+
echo "CONFIG_JSON_ADD_IMAGE_INFO=y" >> .config

0 commit comments

Comments
 (0)