added EKS perf test

Author: jreadey

tests/perf/eksperf/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11.9
+RUN pip install git+https://github.com/hdfgroup/h5pyd.git
+RUN pip install pyyaml
+RUN pip install h5py
+RUN pip install s3fs
+COPY entrypoint.sh /
+RUN chmod 555 /entrypoint.sh
+RUN mkdir /app
+COPY read_shots.py /app
+COPY config.py /app
+COPY config.yml /app
+WORKDIR /app
+ENTRYPOINT ["/entrypoint.sh"]

tests/perf/eksperf/README.md

@@ -0,0 +1,4 @@
+This folder contains files used to create a EKS based performance test.
+
+Based on fusion data from the cmod reactor which is not yet public.
+Will update once the data files have been published.

tests/perf/eksperf/build_image.sh

@@ -0,0 +1 @@
+docker build -t psfc/hsperftest .

tests/perf/eksperf/config.py

@@ -0,0 +1,121 @@
+import os
+import sys
+import yaml
+
+cfg = {}
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def debug(*args, **kwargs):
+    # can't use log.debug since that calls back to cfg
+    if "LOG_LEVEL" in os.environ and os.environ["LOG_LEVEL"] == "DEBUG":
+        print(*args, **kwargs)
+   
+def _load_cfg():    
+    # load config yaml
+    yml_file = None
+    config_dirs = []
+    # check if there is a command line optionfor config directory
+    for i in range(1, len(sys.argv)):
+        #print(i, sys.argv[i])
+        if sys.argv[i].startswith("--config-dir"):
+            # use the given directory
+            arg = sys.argv[i]
+            config_dirs.append(arg[len("--config-dir")+1:])  # return text after option string   
+            debug(f"got cmd line override for config-dir: {config_dirs[0]}")
+            break
+    if not config_dirs and "CONFIG_DIR" in os.environ:
+        config_dirs.append(os.environ["CONFIG_DIR"])
+        debug(f"got environment override for config-dir: {config_dirs[0]}")
+    if not config_dirs:
+        config_dirs = [".", "/config", "/etc/hsds/"]  # default locations
+    for config_dir in config_dirs:
+        file_name = os.path.join(config_dir, "config.yml")
+        debug("checking config path:", file_name)
+        if os.path.isfile(file_name):
+            yml_file = file_name
+            break
+    if not yml_file:
+        msg = "unable to find config file"
+        eprint(msg)
+        raise FileNotFoundError(msg)
+    debug(f"_load_cfg with '{yml_file}'")
+    try:
+        with open(yml_file, "r") as f:
+            yml_config = yaml.safe_load(f)
+    except yaml.scanner.ScannerError as se:
+        msg = f"Error parsing config.yml: {se}"
+        eprint(msg)
+        raise KeyError(msg)
+
+    # load override yaml
+    yml_override = None
+    if "CONFIG_OVERRIDE_PATH" in os.environ:
+        override_yml_filepath = os.environ["CONFIG_OVERRIDE_PATH"]
+    else:
+        override_yml_filepath = "/config/override.yml"
+    if os.path.isfile(override_yml_filepath):
+        debug(f"loading override configuation: {override_yml_filepath}")
+        try:
+            with open(override_yml_filepath, "r") as f:
+                yml_override = yaml.safe_load(f)
+        except yaml.scanner.ScannerError as se:
+            msg = f"Error parsing '{override_yml_filepath}': {se}"
+            eprint(msg)
+            raise KeyError(msg)
+        
+
+    # apply overrides for each key and store in cfg global
+    for x in yml_config:
+        cfgval = yml_config[x]
+        # see if there is a command-line override
+        option = '--'+x+'='
+        override = None
+        for i in range(1, len(sys.argv)):
+            if sys.argv[i].startswith(option):
+                # found an override
+                arg = sys.argv[i]
+                override = arg[len(option):]  # return text after option string   
+                debug(f"got cmd line override for {x}")
+                 
+            
+        # see if there are an environment variable override
+        if override is None and x.upper() in os.environ:
+            override = os.environ[x.upper()]
+            debug(f"got env value override for {x} ")
+
+        # see if there is a yml override
+        if override is None and yml_override and x in yml_override:
+            override = yml_override[x]
+            debug(f"got config override for {x}")
+
+        if override:
+            if cfgval is not None:
+                try:
+                    override = type(cfgval)(override) # convert to same type as yaml
+                except ValueError as ve:
+                    msg = f"Error applying command line override value for key: {x}: {ve}"
+                    eprint(msg)
+                    # raise KeyError(msg)
+            cfgval = override # replace the yml value
+
+        if isinstance(cfgval, str) and len(cfgval) > 1 and cfgval[-1] in ('g', 'm', 'k') and cfgval[:-1].isdigit():
+            # convert values like 512m to corresponding integer
+            u = cfgval[-1]
+            n = int(cfgval[:-1])
+            if u == 'k':
+                cfgval =  n * 1024
+            elif u == 'm':
+                cfgval = n * 1024*1024
+            else: # u == 'g'
+                cfgval = n * 1024*1024*1024
+        cfg[x] = cfgval
+
+def get(x):
+    if not cfg:
+        _load_cfg()
+    if x not in cfg:
+        raise KeyError(f"config value {x} not found")
+    return cfg[x]

tests/perf/eksperf/config.yml

@@ -0,0 +1,8 @@
+# bucket loader configuration
+inventory_domain: /home/mdsplus/eksperftest/s3shots.h5  # the HSDS domain used to store the list of domains
+hs_username: null  # This value should be supplied by a Kubernetes secret or environment variable
+hs_password: null  # This value should be supplied by a Kubernetes secret or environment variable
+public_read: true  # make new domains public read, if false, only the hs_username user will have access
+hs_global: http://10.100.32.2   # e.g: http://10.100.32.2  # run: set the IP here to the CLUSTER_IP you get when running kubectl get service hsds
+hs_local:  http://localhost:5101  # e.g: http://localhost:5101  # this is the pod-local address, set to null if running loader outside of kubernetes
+log_level: DEBUG

tests/perf/eksperf/docker_run.sh

@@ -0,0 +1 @@
+docker run --rm -e HS_GLOBAL=$HS_GLOBAL -e HS_LOCAL=$HS_LOCAL -e HS_USERNAME=$HS_USERNAME -e HS_PASSWORD=$HS_PASSWORD  -it psfc/hsperftest

tests/perf/eksperf/dump_inventory.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+import sys
+import h5pyd
+import config
+
+def formatTime(timestamp):
+    # local_timezone = tzlocal.get_localzone() # get pytz timezone
+    local_time = datetime.fromtimestamp(timestamp) # , local_timezone)
+    return local_time
+
+if len(sys.argv) > 1:
+    if sys.argv[1] in ("-h", "--help"):
+        print(f"usage: python {sys.argv[0]} [inventory_domain]")
+        sys.exit(0)
+    else:
+        inventory_domain = sys.argv[1]
+else:
+    inventory_domain = config.get("inventory_domain")  
+
+f = h5pyd.File(inventory_domain, "r")
+print(f"{inventory_domain} found, owner: {f.owner}, last madified: {datetime.fromtimestamp(f.modified)}")
+print("Contents")
+print("\tfilename\tStart\tDone\tRuntime\tStatus\tPod")
+print("-"*160)
+table = f["inventory"]
+num_succeeded = 0
+num_failed = 0
+num_inprogress = 0
+num_not_started = 0
+
+for row in table:
+    filename = row[0].decode('utf-8')
+    if row[1]:
+        start = formatTime(row[1])
+    else:
+        start = 0
+    if row[2]:
+        stop = formatTime(row[2])
+    else:
+        stop = 0
+    rc = row[3]
+    podname = row[4].decode('utf-8')
+    if row[2] > 0:
+        runtime = f"{int(row[2] - row[1]) // 60:4d}m {(row[2] - row[1]) % 60:2}s"
+    else:
+        runtime = "0"
+
+    if row[1] == 0:
+        num_not_started += 1
+    elif row[2] > 0:
+        if rc == 0:
+            num_succeeded += 1
+        else:
+            num_failed += 1
+    else:
+        num_inprogress += 1
+
+    
+    print(f"{filename}\t{start}\t{stop}\t{runtime}\t{rc}\t{podname}")
+
+print("="*80)    
+print(f"{table.nrows} rows")
+print(f"succeeded: \t{num_succeeded}")
+print(f"failed: \t {num_failed}")
+print(f"in progress: \t {num_inprogress}")
+print(f"not started: \t {num_not_started}")

tests/perf/eksperf/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+echo "hsperf entrypoint"
+export PYTHONUNBUFFERED="1"
+cd /app
+python read_shots.py 0
+echo "unexpected exit of read_shots.py" 

tests/perf/eksperf/get_pods.sh

@@ -0,0 +1 @@
+kubectl -n hsperf get pods