Merge pull request #14 from padovan/standardize-trees-name

ingester: standardize tree names across origins

Author: padovan

Dockerfile.ingester

@@ -9,6 +9,7 @@ RUN git clone https://github.com/kernelci/kcidb.git && \
     pip install -e .
 
 COPY ingester/ingester.py /app/ingester.py
+COPY data/trees.yml /app/trees.yml
 
 CMD ["python", "/app/ingester.py", "--spool-dir", "/app/spool"]
 

data/trees.yml

@@ -0,0 +1,150 @@
+trees:
+  aaptel:
+    url: "https://github.com/aaptel/linux.git"
+
+  amlogic:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git"
+
+  android:
+    url: 'https://android.googlesource.com/kernel/common'
+
+  ardb:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git"
+
+  arm64:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git"
+
+  arnd:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/arnd/playground.git"
+
+  broonie-misc:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/misc.git"
+
+  broonie-regmap:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git"
+
+  broonie-regulator:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git"
+
+  broonie-sound:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git"
+
+  broonie-spi:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git"
+
+  chrome-platform:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git"
+
+  chromiumos:
+    url: "https://chromium.googlesource.com/chromiumos/third_party/kernel.git"
+
+  cip:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git"
+
+  clk:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git"
+
+  collabora-chromeos-kernel:
+    url: 'https://gitlab.collabora.com/google/chromeos-kernel.git'
+
+  efi:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git"
+
+  hyperv:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git"
+
+  kernelci:
+    url: "https://github.com/kernelci/linux.git"
+
+  khilman:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux.git"
+
+  krzysztof:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git"
+
+  kselftest:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git'
+
+  lee-backlight:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git"
+
+  lee-mfd:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git"
+
+  linusw:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-gpio.git"
+
+  linux-pci:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git"
+
+  mainline:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
+
+  media-committers:
+    url: 'https://gitlab.freedesktop.org/linux-media/media-committers.git'
+
+  media:
+    url: 'https://git.linuxtv.org/media.git'
+
+  mediatek:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git'
+
+  net-next:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git"
+
+  netdev-testing:
+    url: "https://github.com/linux-netdev/testing.git"
+
+  next:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git'
+
+  omap:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git"
+
+  pm:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git"
+
+  qcom:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git"
+
+  renesas:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git"
+
+  riscv:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git"
+
+  robh:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git"
+
+  rppt:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git"
+
+  sashal-next:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/sashal/linus-next.git"
+
+  soc:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git"
+
+  stable-rc:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git'
+
+  stable-rt:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-stable-rt.git'
+
+  stable:
+    url: 'https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git'
+
+  tegra:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git"
+
+  thermal:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git"
+
+  tip:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git"
+
+  ulfh:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git"
+
+  vireshk:
+    url: "https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/linux.git"

ingester/ingester.py

@@ -22,6 +22,7 @@
 import json
 import time
 import logging
+import yaml
 
 # default database
 DATABASE = "postgresql:dbname=kcidb user=kcidb password=kcidb host=localhost port=5432"
@@ -60,8 +61,34 @@ def move_file_to_failed_dir(filename, failed_dir):
         print(f"Error moving file {filename} to failed directory: {e}")
         raise e
 
+TREES_FILE = "/app/trees.yml"
 
-def ingest_submissions(spool_dir, db_client=None):
+def load_trees_name():
+    with open(TREES_FILE, "r", encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+
+    trees_name = {
+        v["url"]: tree_name
+        for tree_name, v in data.get("trees", {}).items()
+    }
+
+    return trees_name
+
+
+def standardize_trees_name(input_data, trees_name):
+    """ Standardize tree names in input data using the provided mapping """
+
+    for checkout in input_data.get("checkouts", []):
+        git_url = checkout.get("git_repository_url")
+        if git_url in trees_name:
+            correct_tree = trees_name[git_url]
+            if checkout.get("tree_name") != correct_tree:
+                checkout["tree_name"] = correct_tree
+
+    return input_data
+
+
+def ingest_submissions(spool_dir, trees_name, db_client=None):
     failed_dir = os.path.join(spool_dir, "failed")
     archive_dir = os.path.join(spool_dir, "archive")
     if db_client is None:
@@ -89,6 +116,7 @@ def ingest_submissions(spool_dir, db_client=None):
                     logger.info(f"File size: {fsize}")
                 try:
                     data = json.loads(f.read())
+                    data = standardize_trees_name(data, trees_name)
                     data = io_schema.validate(data)
                     data = io_schema.upgrade(data, copy=False)
                     db_client.load(data)
@@ -147,10 +175,11 @@ def main():
     args = parser.parse_args()
     logger.info("Starting ingestion process...")
     verify_spool_dirs(args.spool_dir)
+    trees_name = load_trees_name()
     get_db_credentials()
     db_client = get_db_client(DATABASE)
     while True:
-        ingest_submissions(args.spool_dir, db_client)
+        ingest_submissions(args.spool_dir, trees_name, db_client)
         time.sleep(1)
 
 if __name__ == "__main__":