New website

Author: m-mohr

.github/build-script/create_data.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+from jinja2 import Template
+import datetime
+import logging
+import json
+import re
+import sys
+import yaml
+
+logger = logging.getLogger(__name__)
+
+def get_externals() -> list:
+  data = []
+
+  return data
+
+def main() -> bool:
+  logger.info(f"Reading lists of datasets")
+
+  catalog = json.loads(Path('./stac/catalog.json').read_text())
+  data = list(filter(lambda link: link["rel"] == "child", catalog["links"]))
+
+  for i, d in enumerate(data):
+    title = re.sub(r"^Field\s+boundaries\s+for\s+", "", d["title"])
+    title = title[0].upper() + title[1:]
+    data[i]["title"] = title
+  
+  config = yaml.safe_load(Path("./data/external.yaml").read_text())
+  for r in config["external"]:
+    data.append(r)
+
+  data.sort(key = lambda x: x["title"])
+  count = len(data)
+  now = datetime.datetime.now(datetime.timezone.utc).strftime("%b %d %Y, %H:%M %Z")
+  template = Template(Path("./data/index.md.jinja").read_text())
+  content = template.render(data=data, updated=now, count=count)
+  with open("./data/index.md", "w", encoding="utf-8") as f:
+    f.write(content)
+
+  sys.exit(0)
+
+
+if __name__ == "__main__":
+  main()

.github/build-script/create_overviews.py

@@ -0,0 +1,93 @@
+from os import error
+from pathlib import Path
+from jinja2 import Template
+import datetime
+import logging
+import requests
+import os
+import sys
+import yaml
+
+logger = logging.getLogger(__name__)
+
+headers = {}
+env = dict(os.environ)
+if "GITHUB_TOKEN" in env:
+  headers["Authorization"] = " ".join(["Bearer", os.environ["GITHUB_TOKEN"]])
+else:
+  logger.warning("No GITHUB_TOKEN found in env")
+
+def unslugify(s: str) -> str:
+  return s.replace("-", " ").replace("_", " ").title()
+
+def get_repo(response) -> dict:
+  data = {
+    "title": unslugify(response["name"]),
+    "url": response["html_url"],
+    "description": response["description"]
+  }
+  if response["archived"]:
+    data["description"] = "**DEPRECATED.** " + data["description"]
+  return data
+
+def get_repos() -> list:
+  logger.info(f"Reading list of repositories")
+
+  with requests.get("https://api.github.com/users/fiboa/repos?per_page=1000", headers=headers) as site:
+    repos = site.json()
+  return repos
+
+def filter_repos(repos, type) -> list:
+  data = []
+  try:
+    for repo in repos:
+      if not isinstance(repo, dict):
+        logger.error(f"response invalid")
+        continue
+      if repo["is_template"] or repo["visibility"] !=	"public" or type not in repo["topics"]:
+        continue
+      data.append(get_repo(repo))
+  except error as e:
+    logger.error(f"fiboa org not available: {e}")
+  return data
+
+def get_externals(base) -> list:
+  data = []
+  config = yaml.safe_load((base / "external.yaml").read_text())
+  for r in config["github"]:
+    try:
+      logger.info(f"Reading community GitHub repos individually")
+      with requests.get(f"https://api.github.com/repos/{r['org']}/{r['repo']}", headers=headers) as repo:
+        data.append(get_repo(repo.json()))
+    except error as e:
+      logger.error(f"community repo not available: {e}")
+
+  for r in config["external"]:
+    data.append(r)
+
+  return data
+
+def create_overview(repos, folder, label):
+  print(f"Generating {folder}")
+  base = Path(f"./{folder}/")
+  data = filter_repos(repos, label)
+  data.extend(get_externals(base))
+  data.sort(key = lambda x: x["title"])
+  count = len(data)
+  now = datetime.datetime.now(datetime.timezone.utc).strftime("%b %d %Y, %H:%M %Z")
+  template = Template((base / "index.md.jinja").read_text())
+  target = base / "index.md"
+  content = template.render(data=data, updated=now, count=count)
+
+  with open(target, "w", encoding="utf-8") as f:
+    f.write(content)
+  
+def main():
+  repos = get_repos()
+  create_overview(repos, "extensions", "extension")
+  create_overview(repos, "software", "software")
+  sys.exit(0)
+
+
+if __name__ == "__main__":
+  main()

.github/build-script/requirements.txt

@@ -0,0 +1,3 @@
+requests
+jinja2
+pyyaml

.github/workflows/build.yml

@@ -0,0 +1,77 @@
+name: Build and Deploy Jekyll Site
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # at 00:00 everyday
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      
+    - name: Setup Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.1'
+        bundler-cache: true
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+        
+    - name: Setup Pages
+      id: pages
+      uses: actions/configure-pages@v4
+
+    - name: Update extensions / software
+      run: |
+        pip install -r .github/build-script/requirements.txt
+        python .github/build-script/create_overviews.py
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Update datasets
+      run: python .github/build-script/create_data.py
+      
+    - name: Build Jekyll site
+      run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+      env:
+        JEKYLL_ENV: production
+
+    - name: Build Map
+      run: |
+        cd map
+        npm ci
+        npm run build
+        cd ..
+        rm -r ./_site/map/*
+        cp -r ./map/dist/* ./_site/map
+
+    - name: Upload artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: ./_site
+        
+    - name: Deploy to GitHub Pages
+      if: github.ref == 'refs/heads/main'
+      id: deployment
+      uses: actions/deploy-pages@v4

.github/workflows/update.yml

@@ -0,0 +1,33 @@
+# Jekyll build files
+_site/
+.sass-cache/
+.jekyll-cache/
+.jekyll-metadata
+
+# Bundler files
+vendor/
+.bundle/
+Gemfile.lock
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Node modules (if using npm)
+node_modules/
+
+# Local development
+.env
+/extensions/index.md
+/software/index.md

.gitignore

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "github-pages", group: :jekyll_plugins