Auto renew certificates and add ignore list (#44)

* refactor and automatically renew credentials older than 90 days

* added --dry-run flag

* added ignorefile

* update deploy script to allow git pull to fail

Author: seanshahkarami

bk-api/bk_api.py

@@ -894,13 +894,11 @@ def deploy_wes(node_id, this_debug, force=False):
             "/config/node-private-git-repo-key/node-private-git-repo-key not found, skipping"
         )
 
-    final_command = "./update-stack.sh"
+    final_command = "/opt/waggle-edge-stack/kubernetes/update-stack.sh"
     if FAKE_DEPLOYMENT:
         final_command = 'echo "This fake deployment was successful"'
 
-    deploy_script = (
-        """\
-#!/bin/sh
+    deploy_script = f"""#!/bin/sh
 ### This script was generated by beekeeper ###
 set -e
 set -x
@@ -913,17 +911,12 @@ def deploy_wes(node_id, this_debug, force=False):
     git clone https://github.com/waggle-sensor/waggle-edge-stack.git
 fi
 
-cd /opt/waggle-edge-stack
-git pull origin main
-#git checkout tags/v.1.0
-
-cd /opt/waggle-edge-stack/kubernetes
+# allow pull to fail so we can still renew certificates on modified repo
+git -C /opt/waggle-edge-stack pull origin main || true
 
+{final_command}
 """
-        + final_command
-    )
 
-    # return {"result": "C"}
     try:
         node_ssh_with_logging(node_id, "cat > /tmp/deploy.sh", input_str=deploy_script)
         node_ssh_with_logging(node_id, "sh /tmp/deploy.sh")

bk-deploy-manager/deploy_manager.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import datetime
 import logging
 import os
@@ -8,20 +9,40 @@
 
 import dateutil.parser
 import requests
+from dataclasses import dataclass
+from typing import Optional
+from pathlib import Path
+
+
+@dataclass
+class Node:
+    id: str
+    beehive: Optional[str]
+    registered_at: Optional[datetime.datetime]
+    deployed_wes_at: Optional[datetime.datetime]
+
+
+@dataclass
+class Candidate:
+    node: dict
+    renew_credentials: bool
 
-logging.basicConfig(level=logging.INFO)
 
 BEEKEEPER_URL = os.getenv("BEEKEEPER_URL", "http://localhost:5000")
 
 
 # example input 2021-11-19T02:07:22
 # returns datetime.datetime
-def parseTime(timestamp):
+def parse_datetime(timestamp: Optional[str]) -> Optional[datetime.datetime]:
+    if timestamp == None:
+        return None
+    if timestamp == "":
+        return None
     # dateutil.parser.isoparse('2008-09-03T20:56:35.450686')
     return dateutil.parser.isoparse(timestamp)
 
 
-def get_candidates():
+def get_nodes() -> list[Node]:
     if BEEKEEPER_URL == "":
         logging.error(f"BEEKEEPER_URL not defined")
         sys.exit(1)
@@ -38,61 +59,114 @@ def get_candidates():
     if resp.status_code != 200:
         raise Exception(f"status_code: {resp.status_code} body: {resp.text}")
 
-    nodes = resp.json()
+    items = resp.json()["data"]
+
+    nodes = []
+
+    for item in items:
+        node_id = item["id"]
+        beehive = item.get("beehive")
+        if beehive == "":
+            beehive = None
+        registered_at = parse_datetime(item.get("registration_event"))
+        deployed_wes_at = parse_datetime(item.get("wes_deploy_event"))
+        nodes.append(
+            Node(
+                id=node_id,
+                beehive=beehive,
+                registered_at=registered_at,
+                deployed_wes_at=deployed_wes_at,
+            )
+        )
+
+    return nodes
+
+
+def get_candidates(ignorelist: list[str]) -> list[Candidate]:
+    nodes = get_nodes()
 
     candidates = []
 
-    if not "data" in nodes:
-        raise Exception("Field data missing")
-
-    for n in nodes["data"]:
-        node_id = n["id"]
-        registration_event = n.get("registration_event")
-        wes_deploy_event = n.get("wes_deploy_event")
-        # print("id: "+node_id)
-        # print("wes_deploy_event: "+n["wes_deploy_event"])
-        if registration_event in ["", None]:
-            logging.info("node %s is not registered", node_id)
+    for node in nodes:
+        if node.id in ignorelist:
+            logging.info("node %s in ignorelist - skipping", node.id)
+            continue
+
+        if node.registered_at is None:
+            logging.info("node %s is not registered", node.id)
+            continue
+
+        if node.beehive is None:
+            logging.info("node %s does not belong to a beehive", node.id)
             continue
 
-        if n.get("beehive") in ["", None]:
-            logging.info(f"node {node_id} does not belong to a beehive")
+        if node.deployed_wes_at is None:
+            logging.info(
+                "scheduling node %s for wes deployment (reason: no previous deployment)",
+                node.id,
+            )
+            candidates.append(Candidate(node=node, renew_credentials=False))
             continue
 
-        if wes_deploy_event in ["", None] or parseTime(registration_event) >= parseTime(
-            wes_deploy_event
-        ):
+        # reregistered nodes also need wes redeployed
+        if node.registered_at >= node.deployed_wes_at:
+            logging.info(
+                "scheduling node %s for wes deployment (reason: node reregistered)",
+                node.id,
+            )
+            candidates.append(Candidate(node=node, renew_credentials=True))
+            continue
+
+        # automatically redeploy with renewed credentials periodically
+        deployed_wes_age = datetime.datetime.now() - node.deployed_wes_at
+
+        if deployed_wes_age >= datetime.timedelta(days=90):
             logging.info(
-                f"scheduling node {node_id} for wes deployment (reason: no previous deployment or re-registered node)"
+                "scheduling node %s for wes deployment (reason: renew certificates - %d days old)",
+                node.id,
+                deployed_wes_age.days,
             )
-            candidates.append(n)
+            candidates.append(Candidate(node=node, renew_credentials=True))
             continue
 
-        logging.info(f"node {node_id} needs no deployment")
+        logging.info("node %s needs no deployment", node.id)
+
+    # Q(sean) Is there ever a time where we wouldn't want to renew credentials? As long as
+    # there's some rate limit on how fast we're generating them, it seems like just renewing
+    # is a much simpler management strategy.
 
     return candidates
 
 
-def try_wes_deployment(candidates):
+def try_wes_deployment(candidates: list[Candidate], dry_run: bool):
     success_count = 0
 
     for candidate in candidates:
         try:
-            deploy_wes_to_candidate(candidate)
+            deploy_wes_to_candidate(candidate, dry_run)
             success_count += 1
         except KeyboardInterrupt:
             return
         except Exception:
             logging.exception("deploy_wes_to_candidate failed")
-        time.sleep(2)
 
     logging.info(f"{success_count} out of {len(candidates)} successful.")
     logging.info("done")
 
 
-def deploy_wes_to_candidate(candidate):
-    node_id = candidate["id"]
-    url = f"{BEEKEEPER_URL}/node/{node_id}"
+def deploy_wes_to_candidate(candidate: Candidate, dry_run: bool):
+    node = candidate.node
+
+    if candidate.renew_credentials:
+        logging.info("deploying to candidate %s with renewed credentials", node.id)
+        url = f"{BEEKEEPER_URL}/node/{node.id}?force=true"
+    else:
+        logging.info("deploying to candidate %s", node.id)
+        url = f"{BEEKEEPER_URL}/node/{node.id}"
+
+    if dry_run:
+        return
+
     resp = requests.post(url, json={"deploy_wes": True})
     resp.raise_for_status()
     result = resp.json()
@@ -103,20 +177,51 @@ def deploy_wes_to_candidate(candidate):
 
 
 def main():
-    logging.info("Starting...")
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="enable verbose logging",
+    )
+    parser.add_argument(
+        "--ignorefile",
+        help="path of ignorefile",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="get and log candidates but do not deploy wes",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.debug else logging.INFO,
+        format="%(asctime)s %(message)s",
+        datefmt="%Y/%m/%d %H:%M:%S",
+    )
+
+    if args.ignorefile:
+        logging.info("loading ignorefile %s", args.ignorefile)
+        ignorefile = Path(args.ignorefile)
+        ignorelist = [s.upper().rjust(16, "0") for s in ignorefile.read_text().split()]
+    else:
+        logging.info("no ignorefile provided")
+        ignorelist = []
+
     while True:
         candidates = []
+
         try:
-            candidates = get_candidates()
+            logging.info("getting candidates")
+            candidates = get_candidates(ignorelist)
         except Exception as e:
             logging.error(f"error: get_candidates returned: {str(e)}")
 
         if len(candidates) == 0:
             logging.info("no candidates for wes deployment found")
         else:
-            logging.info("candidates:")
-            logging.info(candidates)
-            try_wes_deployment(candidates)
+            logging.info("deploying to candidates")
+            try_wes_deployment(candidates, args.dry_run)
 
         logging.info("waiting 5 minutes...")
         time.sleep(5 * 60)