[docsy] Add external link checking, fix links, add cache

Author: chalin

.htmltest.yml

@@ -1,10 +1,50 @@
+CacheExpires: 9000h # ~ 12 months
 DirectoryPath: public
+TestFilesConcurrently: true
 CheckDoctype: false # Sadly, this is false only because of `static/google*.html`
+CheckMailto: false
 IgnoreAltMissing: true # FIXME
 IgnoreDirectoryMissingTrailingSlash: true # FIXME
 IgnoreDirs:
 IgnoreEmptyHref: true # FIXME
 IgnoreInternalEmptyHash: true # FIXME
 IgnoreInternalURLs: # list of paths
 IgnoreURLs: # list of regexs of paths or URLs to be ignored
+  # Ignore Docsy-generated GitHub links for now
+  - ^https?://github\.com/.*?/.*?/(new|edit)/ # view-page, edit-source etc
+
+  # TUF
   - ^/specification/
+  - ^https://cse.google.com
+
+  # FIXME: 4XXs reported by checker, which we ignore until we have time to fix them
+  # Get "https://events17.linuxfoundation.org/events/kubecon-and-cloudnativecon-north-america/program/schedule": x509: certificate is valid for *.cass.oregonstate.edu, not events17.linuxfoundation.org --- resources/news/index.html
+  - ^https://events17.linuxfoundation.org/events/kubecon-and-cloudnativecon-north-america/program/schedule
+  # Non-OK status: 403 --- resources/news/index.html
+  - ^https://schd.ws/hosted_files/linuxconcontainerconeurope2016/50/When%20the%20going%20gets%20tough%2C%20get%20TUF%20going%21%20Linuxcon%20EU.pdf
+  # Non-OK status: 403 --- resources/news/index.html
+  - ^https://www.forbes.com/sites/.../uptane-will-protect-your-connected-car-from-hackers
+  # Non-OK status: 403 --- resources/news/index.html
+  - ^https://www.tmcnet.com/usubmit/2019/05/28/8963021.htm
+  # Non-OK status: 404 --- community/adoptions/index.html
+  - ^https://github.com/bottlerocket-os/bottlerocket/tree/develop/sources/updater
+  # Non-OK status: 404 --- resources/news/index.html
+  - ^https://www.d2pmagazine.com/2020/04/02/6099/
+  # Non-OK status: 404 --- resources/news/index.html
+  - ^https://www.just-auto.com/news/here-and-uptane-team-on-automotiveiot-security_id188912.aspx
+  # Non-OK status: 404 --- resources/news/index.html
+  - ^https://www.linuxfoundation.org/cloud-containers-virtualization/cncf-host-two-security-projects-notary-tuf-specification/
+  # Non-OK status: 404 --- resources/news/index.html
+  - ^https://www.ustream.tv/recorded/64499822#t=1h54m0s
+  # Non-OK status: 503 --- resources/news/index.html
+  - ^https://www.airbiquity.com
+  # Non-OK status: 503 --- resources/news/index.html
+  - ^https://www.airbiquity.com/news/press-releases/airbiquity-bolsters-otamatictm-security-and-data-analytic-features-latest-over-air-ota-software-and-data-management-offering-aut
+  # Non-OK status: 503 --- resources/news/index.html
+  - ^https://www.airbiquity.com/news/press-releases/airbiquity-otamatic-named-2017-new-product-year-business-intelligence-group
+  # Non-OK status: 503 --- resources/news/index.html
+  - ^https://www.airbiquity.com/news/press-releases/airbiquity-otamatic-named-2017-new-product-year-business-intelligence-group
+  # request exceeded our ExternalTimeout --- docs/security/audits/index.html
+  - ^https://www.nccgroup.trust/globalassets/our-research/us/public-reports/2017/ncc-group-kolide-the-update-framework-security-assessment.pdf
+  # request exceeded our ExternalTimeout --- resources/news/index.html
+  - ^http://www.enterprisecloudnews.com/author.asp

Makefile

@@ -1,6 +1,12 @@
+# Set REFCACHE to another value to disable htmltest refcache-file manipulation
+REFCACHE?=refcache
 HTMLTEST_DIR=tmp
 HTMLTEST?=htmltest # Specify as make arg if different
 HTMLTEST_ARGS?=--skip-external
+LINK_CACHE_FILE?=refcache.json
+LINK_CACHE_FILE_DEST_DIR?=static
+LINK_CACHE_FILE_SRC_DIR?=$(HTMLTEST_DIR)/.htmltest
+OTEL_GEN_REPO?=../$(shell basename $(shell pwd)).g
 
 # Use $(HTMLTEST) in PATH, if available; otherwise, we'll get a copy
 ifeq (, $(shell which $(HTMLTEST)))
@@ -10,7 +16,32 @@ GET_LINK_CHECKER_IF_NEEDED=get-link-checker
 endif
 endif
 
-check-links: $(GET_LINK_CHECKER_IF_NEEDED)
+default:
+	@echo "Make what? Target list:\n"
+	@make -rpn | grep '^[a-z]\S*:' | sed 's/://' | sort
+
+$(LINK_CACHE_FILE_SRC_DIR):
+	mkdir -p $(LINK_CACHE_FILE_SRC_DIR)
+
+refcache-restore: $(LINK_CACHE_FILE_SRC_DIR)
+ifeq (refcache, $(REFCACHE))
+	cp $(LINK_CACHE_FILE_DEST_DIR)/$(LINK_CACHE_FILE) $(LINK_CACHE_FILE_SRC_DIR)/
+else
+	@echo "SKIPPING refcache-restore"
+endif
+
+refcache-save: $(LINK_CACHE_FILE_SRC_DIR)/$(LINK_CACHE_FILE)
+ifeq (refcache, $(REFCACHE))
+	cp $(LINK_CACHE_FILE_SRC_DIR)/$(LINK_CACHE_FILE) $(LINK_CACHE_FILE_DEST_DIR)/
+	npm run _prettier:any -- --write $(LINK_CACHE_FILE_DEST_DIR)/$(LINK_CACHE_FILE)
+else
+	@echo "SKIPPING refcache-save"
+endif
+
+check-links: $(GET_LINK_CHECKER_IF_NEEDED) \
+	refcache-restore check-links-only refcache-save
+
+check-links-only:
 	$(HTMLTEST) $(HTMLTEST_ARGS)
 
 clean:
@@ -19,3 +50,16 @@ clean:
 get-link-checker:
 	rm -Rf $(HTMLTEST_DIR)/bin
 	curl https://htmltest.wjdp.uk | bash -s -- -b $(HTMLTEST_DIR)/bin
+
+# For local development, create `public` either as a symlink to a given git repo
+# (if it exists), or an empty git repo. This is for the purpose of tracking
+# build changes.
+public:
+	@if [ -e "$(OTEL_GEN_REPO)" ]; then \
+		set -x; ln -s $(OTEL_GEN_REPO) public; \
+	elif [ -z "$(CI)" ]; then \
+		set -x; git init public; \
+	fi
+
+ls-public:
+	if [ -e public ]; then ls -ld public; fi

content/en/docs/project/_index.md

@@ -27,7 +27,7 @@ To learn how project decisions are made, and for a more detailed explanation of
 the project roles used below, see [Governance].
 
 [contributed]:
-  https://github.com/theupdateframework/python-tuf/blob/develop/docs/AUTHORS.txt
+  https://github.com/theupdateframework/python-tuf/blob/develop/README.md#acknowledgements
 [Governance]:
   https://github.com/theupdateframework/specification/blob/master/GOVERNANCE.md
 [Specification]: /specification/latest

layouts/partials/hooks/head-end.html

@@ -1 +1,3 @@
-<script type="text/javascript" async src="https://cse.google.com/cse.js?cx={{ .Site.Params.gcs_engine_id }}"></script>
+{{ with .Site.Params.gcs_engine_id -}}
+  <script type="text/javascript" async src="https://cse.google.com/cse.js?cx={{ . }}"></script>
+{{ end -}}

package.json

@@ -5,7 +5,8 @@
     "_build": "npm run _hugo-dev --",
     "_check:format": "npx prettier --check .",
     "_check:links--warn": "npm run _check:links || (echo; echo 'WARNING: see link-checker output for issues.'; echo)",
-    "_check:links": "HTMLTEST_ARGS='--log-level 1 --skip-external' npm run __check:links",
+    "_check:links:internal": "npm run __check:links",
+    "_check:links": "HTMLTEST_ARGS='--log-level 1' npm run __check:links",
     "_diff:check": "git diff --name-only --exit-code",
     "_filename-error": "echo 'ERROR: the following files violate naming conventions; fix using: `npm run fix:filenames`'; echo; npm run -s _ls-bad-filenames; exit 1",
     "_filenames-to-kebab-case": "find assets content -name '*_*' ! -name '[_.]*' -exec sh -c 'mv \"$1\" \"${1//_/-}\"' _ {} \\;",
@@ -15,13 +16,14 @@
     "_hugo": "hugo --cleanDestinationDir",
     "_ls-bad-filenames": "find assets content -name '*_*' ! -name '[_.]*'",
     "_prepare:docsy": "cd themes/docsy && npm install",
+    "_prettier:any": "npx prettier --ignore-path ''",
     "_serve": "npm run _hugo-dev -- serve --minify --disableFastRender --renderToMemory",
     "build:preview": "npm run _hugo-dev -- --minify --baseURL \"${DEPLOY_PRIME_URL:-http://localhost}\"",
     "build:production": "npm run _hugo -- --minify",
     "build": "npm run _build --",
     "check:filenames": "test -z \"$(npm run -s _ls-bad-filenames)\" || npm run -s _filename-error",
     "check:format": "npm run _check:format || (echo '[help] Run: npm run fix:format'; exit 1)",
-    "check:links:all": "HTMLTEST_ARGS= npm run _check:links",
+    "check:links:internal": "npm run _check:links:internal",
     "check:links": "npm run _check:links",
     "clean": "rm -Rf public",
     "diff:check": "npm run _diff:check || (echo; echo 'WARNING: the files above have not been committed'; echo)",