Use SG's db.json as source of truth

See whatwg/sg#111 for context.

Closes whatwg/sg#75.

Co-authored-by: Domenic Denicola <[email protected]>

Author: annevk

.gitignore

@@ -11,6 +11,13 @@ whatwg.org/principles
 whatwg.org/policies
 whatwg.org/working-mode
 
+# Ignore files generated by convert_sg_db.py
+__pycache__/
+idea.whatwg.org/index.html
+resources.whatwg.org/biblio.json
+spec.whatwg.org/index.html
+whatwg.org/workstreams
+
 # Ignore generated snapshot logos
 resources.whatwg.org/logo*-snapshot.svg
 

convert-policy.py renamed to convert_policy.py

@@ -4,7 +4,8 @@
 import re
 
 
-def parse_link_mapping(link_mapping):
+def obtain_link_mapping():
+    link_mapping = open("sg/policy-link-mapping.txt", "r", encoding="utf-8").read()
     return [line.split('=',1) for line in link_mapping.split("\n") if len(line) > 0]
 
 
@@ -52,9 +53,10 @@ def rewrite_defs(policy_markdown):
     return re.sub(r'<a id=([^>]*)>[*][*]([^*]*)[*][*]</a>', '<dfn id=\\1>\\2</dfn>', policy_markdown)
 
 
-def preprocess_markdown(policy_markdown, mapping_pairs):
+def markdown(policy_markdown, mapping_pairs = []):
     result = apply_link_mapping(policy_markdown, mapping_pairs)
     result = rewrite_defs(result)
+    result = commonmark.commonmark(result)
 
     return result
 
@@ -83,7 +85,7 @@ def markdown_title(policy_markdown):
 
 
 def main():
-    link_mapping_pairs = parse_link_mapping(open("sg/policy-link-mapping.txt", "r", encoding="utf-8").read())
+    link_mapping_pairs = obtain_link_mapping()
     template = open("site-template.html", "r", encoding="utf-8").read()
     for resource, link in link_mapping_pairs:
         if link.startswith("https:"):
@@ -92,9 +94,8 @@ def main():
         policy_markdown = open("sg" + resource[1:].replace("%20", " "), "r", encoding="utf-8").read()
 
         (title, policy_markdown) = markdown_title(policy_markdown)
-        preprocessed_policy_markdown = preprocess_markdown(policy_markdown, link_mapping_pairs)
 
-        policy_html = commonmark.commonmark(preprocessed_policy_markdown)
+        policy_html = markdown(policy_markdown, link_mapping_pairs)
 
         postprocessed_policy_html = postprocess_html(policy_html, template, title)
 

convert_sg_db.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+import json
+from datetime import date
+from convert_policy import markdown, obtain_link_mapping
+
+
+def create_biblio(db):
+    biblio = {}
+
+    for workstream in db["workstreams"]:
+        for standard in workstream["standards"]:
+            reference, contents = standard_or_idea_to_biblio(standard)
+            contents["title"] += " Standard"
+            biblio[reference] = contents
+
+    for idea in db["ideas"]:
+        reference, contents = standard_or_idea_to_biblio(idea)
+        contents["status"] = "Living Idea"
+        biblio[reference] = contents
+
+    for biblio_item in db["biblio"]:
+        reference, contents = biblio_item_to_biblio(biblio_item)
+        biblio[reference] = contents
+
+    string = json.dumps(biblio, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2)
+    return string + "\n"
+
+def standard_or_idea_to_biblio(document):
+    return (document["reference"], {
+        "authors": [author_to_biblio(author) for author in document["authors"]],
+        "href": document["href"],
+        "title": document["name"]
+    })
+
+def biblio_item_to_biblio(item):
+    contents = {
+        "href": item["href"],
+        "title": item["title"]
+    }
+    if len(item["authors"]):
+        contents["authors"] = [author_to_biblio(author) for author in item["authors"]]
+    if "obsoletedBy" in item:
+        contents["obsoletedBy"] = item["obsoletedBy"]
+    return (item["reference"], contents)
+
+def author_to_biblio(person):
+    return person["name"]
+
+
+def create_spec_whatwg_org(db, template):
+    content = "<p>The WHATWG works on a number of technologies that are fundamental parts of the web platform. They are organised somewhat arbitrarily based on the preferences of those editing the standard for those technologies.</p>\n\n"
+    content += "<dl>"
+    for workstream in db["workstreams"]:
+        for standard in workstream["standards"]:
+            content += standard_or_idea_to_html(standard)
+    content += "</dl>\n\n<p>The WHATWG also works on a <a href=\"https://idea.whatwg.org/\">number of ideas</a> that aspire to become standards one day.</p>\n\n"
+
+    template = template.replace("<a href=\"https://spec.whatwg.org/\">Standards</a>", "<a>Standards</a>")
+    return wrap_in_site_template(template, "Standards", content)
+
+def create_idea_whatwg_org(db, template):
+    content = "<p>The WHATWG works on a number of ideas that aspire to become web platform standards.</p>\n\n"
+    content += "<dl>"
+    for idea in db["ideas"]:
+        content += standard_or_idea_to_html(idea)
+    content += "</dl>\n\n"
+
+    return wrap_in_site_template(template, "Ideas", content)
+
+def standard_or_idea_to_html(document):
+    output = "\n <dt><a href=\"{0}\">{1}</a></dt>".format(document["href"], document["name"])
+    return output + "\n <dd>{}</dd>\n".format(markdown(document["description"])[:-1]) # Strip trailing \n
+
+
+def create_workstreams(db, template):
+    introduction = open("sg/Workstreams Introduction.md", "r", encoding="utf-8").read()
+    introduction = introduction.replace("<!-- This ends up being included at https://whatwg.org/workstreams -->\n", "")
+    content = markdown(introduction, obtain_link_mapping())
+    for workstream in db["workstreams"]:
+        content += "\n<h3>{}</h3>".format(workstream["name"])
+        content += """\n<dl class="compact">"""
+        content += "\n <div>"
+        content += "\n  <dt>Scope</dt>"
+        content += "\n  <dd>{}</dd>".format(markdown(workstream["scope"])[3:-5]) # Strip leading <p> and trailing </p>\n
+        content += "\n </div>"
+        content += "\n <div>"
+        content += "\n  <dt>Editor{}</dt>".format("" if len(workstream["editors"]) == 1 else "s")
+        for editor in workstream["editors"]:
+            content += """\n  <dd>{0}, <a href="mailto:{1}">{1}</a></dd>""".format(editor["name"], editor["email"])
+        content += "\n </div>"
+        content += "\n <div>"
+        content += "\n  <dt>Standard{}</dt>".format("" if len(workstream["standards"]) == 1 else "s")
+        for standard in workstream["standards"]:
+            content += """\n  <dd><a href="{0}">{1}</a></dd>""".format(standard["href"], standard["name"])
+        content += "\n </div>"
+        content += "\n</dl>"
+
+    return wrap_in_site_template(template, "Workstreams", content)
+
+
+def wrap_in_site_template(template, title, content):
+    output = template.replace("@TITLE_GOES_HERE@", title)
+    return output.replace("@CONTENT_GOES_HERE@", content)
+
+def main():
+    template = open("site-template.html", "r", encoding="utf-8").read()
+    copyright = """<footer>
+ <p><small>Copyright © {} WHATWG (Apple, Google, Mozilla, Microsoft). This work is licensed under a <a rel="license" href="https://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.</small></p>
+</footer>""".format(str(date.today().year))
+    template = template.replace("@CONTENT_GOES_HERE@", "@CONTENT_GOES_HERE@" + copyright)
+    db = json.load(open("sg/db.json", "r", encoding="utf-8"))
+
+    for filename, content in [
+      ("resources.whatwg.org/biblio.json", create_biblio(db)),
+      ("spec.whatwg.org/index.html", create_spec_whatwg_org(db, template)),
+      ("idea.whatwg.org/index.html", create_idea_whatwg_org(db, template)),
+      ("whatwg.org/workstreams", create_workstreams(db, template))
+    ]:
+        open(filename, "w", encoding="utf-8").write(content)
+
+main()

deploy.sh

@@ -7,7 +7,8 @@ header() {
 
 header "Importing content from whatwg/sg and making it suitable for whatwg.org"
 git clone --depth=1 https://github.com/whatwg/sg sg
-./convert-policy.py
+./convert_policy.py
+./convert_sg_db.py
 rm -rf sg
 echo ""