Merge branch 'main' into line-numbers

Author: vivian-rook

.github/workflows/build-and-push.yaml

@@ -11,6 +11,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: git fetch
+        run: |
+          git fetch
+      - name: git checkout
+        run: |
+          git checkout ${{ github.head_ref }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 

.gitignore

@@ -19,3 +19,4 @@ config.yaml
 terraform.tfstate*
 .terraform*
 tofu/kube.config
+ansible/collections/*

ansible/ansible.cfg

@@ -1,5 +1,13 @@
 [defaults]
-
 # Better error output
 stdout_callback=debug
 stderr_callback=debug
+
+collections_path=./collections/ansible_collections
+
+# we're only using localhost, no need for the warning.
+localhost_warning=False
+
+[inventory]
+# Only using localhost, so no inventory
+inventory_unparsed_warning=False

deploy.sh

@@ -22,13 +22,18 @@ source secrets.sh
 python3 -m venv .venv/deploy
 source .venv/deploy/bin/activate
 pip install ansible==8.1.0 kubernetes==26.1.0
+# install helm diff. Needed to keep helm module idempotent
+helm plugin install https://github.com/databus23/helm-diff || true
 
 cd tofu
 AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu init
 AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu apply # -auto-approve
 export KUBECONFIG=$(pwd)/kube.config
 
 cd ../ansible
+# install collections here to take advantage of ansible.cfg configs
+ansible-galaxy collection install -U kubernetes.core -p ./collections
+
 ansible-playbook quarry.yaml
 #kubectl create namespace quarry --dry-run=client -o yaml | kubectl apply -f -
 #helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml

helm-quarry/prod-config.yaml

@@ -40,6 +40,11 @@ TOOLS_DB_PORT: 3306
 TOOLS_DB_USER: ''
 TOOLS_DB_PASSWORD: ''
 
+QUARRY_P_HOST: 'db'
+QUARRY_P_PORT: 3306
+QUARRY_P_USER: ''
+QUARRY_P_PASSWORD: ''
+
 OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite'
 REDIS_HOST: 'redis'
 REDIS_PORT: 6379

quarry/default_config.yaml

@@ -13,13 +13,17 @@ def __init__(self, config):
 
     def _db_name_mangler(self):
         self.is_tools_db = False
+        self.is_quarry_p = False
         if self.dbname == "":
             raise ReplicaConnectionException(
                 "Attempting connection before a database is selected"
             )
         if "__" in self.dbname and self.dbname.endswith("_p"):
             self.is_tools_db = True
             self.database_p = self.dbname
+        elif self.dbname == "quarry" or self.dbname == "quarry_p":
+            self.is_quarry_p = True
+            self.database_p = "quarry_p"
         elif self.dbname == "meta" or self.dbname == "meta_p":
             self.database_name = "s7"
             self.database_p = "meta_p"
@@ -41,6 +45,8 @@ def _db_name_mangler(self):
     def get_host_name(self):
         if self.is_tools_db:
             return self.config["TOOLS_DB_HOST"]
+        if self.is_quarry_p:
+            return self.config["DB_HOST"]
         if self.config["REPLICA_DOMAIN"]:
             return f"{self.database_name}.{self.config['REPLICA_DOMAIN']}"
         return self.database_name
@@ -62,7 +68,12 @@ def connection(self, db):
         self.dbname = db
         self._db_name_mangler()
         host = self.get_host_name()
-        conf_prefix = "TOOLS_DB" if self.is_tools_db else "REPLICA"
+        if self.is_tools_db:
+            conf_prefix = "TOOLS_DB"
+        elif self.is_quarry_p:
+            conf_prefix = "QUARRY_P"
+        else:
+            conf_prefix = "REPLICA"
         port = self.config[f"{conf_prefix}_PORT"]
         connect_opts = {
             "db": self.database_p,

quarry/web/replica.py

@@ -2,7 +2,7 @@
 
 
 VALID_DB_NAMES = re.compile(
-    r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry|s\d+__\w+_p)$"
+    r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry(?:_p)?|s\d+__\w+_p)$"
 )
 
 

quarry/web/utils/__init__.py

@@ -27,8 +27,13 @@
 
 
 def get_replag(cur):
-    cur.execute("SELECT lag FROM heartbeat_p.heartbeat;")
-    return int(cur.fetchall()[0][0])
+    cur.execute("SELECT * FROM information_schema.tables WHERE table_schema='heartbeat_p' and table_name='heartbeat';")
+    if cur.rowcount:
+        cur.execute("SELECT lag FROM heartbeat_p.heartbeat;")
+        return int(cur.fetchall()[0][0])
+    else:
+        # there is not a heartbeat table on this database
+        return 0
 
 
 @worker_process_init.connect

quarry/web/worker.py

@@ -59,3 +59,9 @@ CREATE TABLE IF NOT EXISTS star(
 CREATE INDEX IF NOT EXISTS star_user_id_index ON star(user_id);
 CREATE INDEX IF NOT EXISTS star_query_id_index ON star(query_id);
 CREATE UNIQUE INDEX IF NOT EXISTS star_user_query_index ON star(user_id, query_id);
+
+CREATE DATABASE IF NOT EXISTS quarry_p CHARACTER SET utf8;
+CREATE VIEW IF NOT EXISTS quarry_p.query AS SELECT * FROM quarry.query;
+CREATE VIEW IF NOT EXISTS quarry_p.query_revision AS SELECT * FROM quarry.query_revision;
+CREATE VIEW IF NOT EXISTS quarry_p.query_run AS SELECT * FROM quarry.query_run;
+CREATE VIEW IF NOT EXISTS quarry_p.star AS SELECT * FROM quarry.star;