docs(getting-started): update stackablectl op install output (#589)

* docs(getting-started): update stackablectl op install output

* chore(getting-started): add warning to generated script

* chore(ruff): formatting

* chore(clippy): resolve warning

* chore(nix): update Cargo.nix

* chore(shellcheck): ignore warnings that are not relevant in this context

---------

Co-authored-by: Nick <[email protected]>

Author: NickLarsenNZ

docs/modules/druid/examples/getting_started/getting_started.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# DO NOT EDIT THE SCRIPT
+# Instead, update the j2 template, and regenerate it for dev:
+# cat <<EOF | jinja2 --format yaml getting_started.sh.j2 -o getting_started.sh
+# helm:
+#   repo_name: stackable-dev
+#   repo_url: https://repo.stackable.tech/repository/helm-dev/
+# versions:
+#   commons: 0.0.0-dev
+#   druid: 0.0.0-dev
+#   hdfs: 0.0.0-dev
+#   listener: 0.0.0-dev
+#   secret: 0.0.0-dev
+#   zk: 0.0.0-dev
+# EOF
+
 # The getting started guide script
 # It uses tagged regions which are included in the documentation
 # https://docs.asciidoctor.org/asciidoc/latest/directives/include-tagged-regions/
@@ -122,10 +137,12 @@ kubectl rollout status --watch statefulset/simple-druid-router-default --timeout
 # end::watch-druid-rollout[]
 
 echo "Starting port-forwarding of port 8888"
+# shellcheck disable=2069 # we want all output to be blackholed
 # tag::port-forwarding[]
 kubectl port-forward svc/simple-druid-router 8888 > /dev/null 2>&1 &
 # end::port-forwarding[]
 PORT_FORWARD_PID=$!
+# shellcheck disable=2064 # we want the PID evaluated now, not at the time the trap is
 trap "kill $PORT_FORWARD_PID" EXIT
 sleep 5
 

docs/modules/druid/examples/getting_started/getting_started.sh.j2

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# DO NOT EDIT THE SCRIPT
+# Instead, update the j2 template, and regenerate it for dev:
+# cat <<EOF | jinja2 --format yaml getting_started.sh.j2 -o getting_started.sh
+# helm:
+#   repo_name: stackable-dev
+#   repo_url: https://repo.stackable.tech/repository/helm-dev/
+# versions:
+#   commons: 0.0.0-dev
+#   druid: 0.0.0-dev
+#   hdfs: 0.0.0-dev
+#   listener: 0.0.0-dev
+#   secret: 0.0.0-dev
+#   zk: 0.0.0-dev
+# EOF
+
 # The getting started guide script
 # It uses tagged regions which are included in the documentation
 # https://docs.asciidoctor.org/asciidoc/latest/directives/include-tagged-regions/
@@ -122,10 +137,12 @@ kubectl rollout status --watch statefulset/simple-druid-router-default --timeout
 # end::watch-druid-rollout[]
 
 echo "Starting port-forwarding of port 8888"
+# shellcheck disable=2069 # we want all output to be blackholed
 # tag::port-forwarding[]
 kubectl port-forward svc/simple-druid-router 8888 > /dev/null 2>&1 &
 # end::port-forwarding[]
 PORT_FORWARD_PID=$!
+# shellcheck disable=2064 # we want the PID evaluated now, not at the time the trap is
 trap "kill $PORT_FORWARD_PID" EXIT
 sleep 5
 

docs/modules/druid/examples/getting_started/install_output.txt

@@ -0,0 +1,6 @@
+Installed commons=0.0.0-dev operator
+Installed secret=0.0.0-dev operator
+Installed listener=0.0.0-dev operator
+Installed zookeeper=0.0.0-dev operator
+Installed hdfs=0.0.0-dev operator
+Installed druid=0.0.0-dev operator

docs/modules/druid/examples/getting_started/install_output.txt.j2

@@ -0,0 +1,6 @@
+Installed commons={{ versions.commons }} operator
+Installed secret={{ versions.secret }} operator
+Installed listener={{ versions.listener }} operator
+Installed zookeeper={{ versions.zookeeper }} operator
+Installed hdfs={{ versions.hdfs }} operator
+Installed druid={{ versions.druid }} operator

docs/modules/druid/pages/getting_started/installation.adoc

@@ -25,14 +25,8 @@ include::example$getting_started/getting_started.sh[tag=stackablectl-install-ope
 
 The tool will show
 
-----
-Installed commons operator
-Installed secret operator
-Installed listener operator
-Installed zookeeper operator
-Installed hdfs operator
-Installed druid operator
-----
+[source]
+include::example$getting_started/install_output.txt[]
 
 TIP: Consult the xref:management:stackablectl:quickstart.adoc[] to learn more about how to use `stackablectl`.
 

rust/operator-binary/src/druid_controller.rs

@@ -1420,7 +1420,7 @@ mod test {
                     druid_segment_cache_property = rg_configmap
                         .data
                         .unwrap()
-                        .get(&RUNTIME_PROPS.to_string())
+                        .get(RUNTIME_PROPS)
                         .unwrap()
                         .to_string();
 

tests/templates/kuttl/authorizer/authcheck.py

@@ -13,24 +13,28 @@ def create_user(user_name):
         auth=("admin", "password1"),
         verify=False,
     )
-    data = f"{{\"password\": \"{user_name}\"}}"
+    data = f'{{"password": "{user_name}"}}'
     headers = {
-        'Content-Type': 'application/json',
+        "Content-Type": "application/json",
     }
     requests.post(
         f"https://{coordinator_host}:{coordinator_port}/druid-ext/basic-security/authentication/db/{authenticator_name}/users/{user_name}/credentials",
         headers=headers,
         data=data,
-        auth=('admin', 'password1'),
+        auth=("admin", "password1"),
         verify=False,
     )
 
 
 if __name__ == "__main__":
     result = 0
 
-    log_level = 'DEBUG'
-    logging.basicConfig(level=log_level, format='%(asctime)s %(levelname)s: %(message)s', stream=sys.stdout)
+    log_level = "DEBUG"
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s %(levelname)s: %(message)s",
+        stream=sys.stdout,
+    )
 
     print("CREATING USERS")
     create_user("alice")

tests/templates/kuttl/commons/healthcheck.py

@@ -6,8 +6,12 @@
 if __name__ == "__main__":
     result = 0
 
-    log_level = 'DEBUG'  # if args.debug else 'INFO'
-    logging.basicConfig(level=log_level, format='%(asctime)s %(levelname)s: %(message)s', stream=sys.stdout)
+    log_level = "DEBUG"  # if args.debug else 'INFO'
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s %(levelname)s: %(message)s",
+        stream=sys.stdout,
+    )
 
     druid_cluster_name = sys.argv[1]
 
@@ -42,7 +46,9 @@
                 if res.status_code == 200 and res.text.lower() == "true":
                     break
                 else:
-                    print(f"Got non 200 status code [{res.status_code}] or non-true response [{res.text.lower()}], retrying attempt no [{count}] ....")
+                    print(
+                        f"Got non 200 status code [{res.status_code}] or non-true response [{res.text.lower()}], retrying attempt no [{count}] ...."
+                    )
             except requests.exceptions.Timeout:
                 print(f"Connection timed out, retrying attempt no [{count}] ....")
             except requests.ConnectionError as e:

tests/templates/kuttl/commons/ingestioncheck-tls.py

@@ -12,11 +12,17 @@
 class DruidClient:
     def __init__(self, cert, verify):
         self.session = requests.Session()
-        self.session.headers.update({'Accept': 'application/json', 'Content-Type': 'application/json'})
-        if os.environ['DRUID_OIDC'] == 'true':
-            if os.environ['DRUID_USER'] == '' or os.environ['DRUID_PASSWORD'] == '':
-                sys.exit("Need to provide basic auth user and password when OIDC is enabled.")
-            self.session.auth = HTTPBasicAuth(os.environ['DRUID_USER'], os.environ['DRUID_PASSWORD'])
+        self.session.headers.update(
+            {"Accept": "application/json", "Content-Type": "application/json"}
+        )
+        if os.environ["DRUID_OIDC"] == "true":
+            if os.environ["DRUID_USER"] == "" or os.environ["DRUID_PASSWORD"] == "":
+                sys.exit(
+                    "Need to provide basic auth user and password when OIDC is enabled."
+                )
+            self.session.auth = HTTPBasicAuth(
+                os.environ["DRUID_USER"], os.environ["DRUID_PASSWORD"]
+            )
         self.session.cert = cert
         self.session.verify = verify
 
@@ -33,7 +39,7 @@ def get_tasks(self, url):
         return response.text
 
     def post_task(self, url, input):
-        response = self.session.post(url, data=open(input, 'rb'))
+        response = self.session.post(url, data=open(input, "rb"))
         assert response.status_code == 200
         return response.text
 
@@ -77,44 +83,46 @@ def query_datasource(self, url, sql, expected, iterations):
     cert = ("/tmp/druid-tls/tls.crt", "/tmp/druid-tls/tls.key")
     verify = "/tmp/druid-tls/ca.crt"
 else:
-    sys.exit("Usage: python ./ingestioncheck.py <namespace> <cluster_name> <no-tls|internal-and-server-tls|internal-and-server-tls-and-tls-client-auth>")
+    sys.exit(
+        "Usage: python ./ingestioncheck.py <namespace> <cluster_name> <no-tls|internal-and-server-tls|internal-and-server-tls-and-tls-client-auth>"
+    )
 
 
 druid = DruidClient(cert, verify)
 
-print('''
+print("""
 Query tasks
-===========''')
+===========""")
 tasks = druid.get_tasks(
     url=f"{protocol}://{druid_cluster_name}-coordinator-default-0.{druid_cluster_name}-coordinator-default.{namespace}.svc.cluster.local:{coordinator_port}/druid/indexer/v1/tasks",
 )
 task_count = len(json.loads(tasks))
-print(f'existing tasks: {task_count}')
+print(f"existing tasks: {task_count}")
 
-print('''
+print("""
 Start ingestion task
-====================''')
+====================""")
 ingestion = druid.post_task(
     url=f"{protocol}://{druid_cluster_name}-coordinator-default-0.{druid_cluster_name}-coordinator-default.{namespace}.svc.cluster.local:{coordinator_port}/druid/indexer/v1/task",
-    input='/tmp/druid-quickstartimport.json'
+    input="/tmp/druid-quickstartimport.json",
 )
 task_id = json.loads(ingestion)["task"]
-url_encoded_taskid = urllib.parse.quote(task_id, safe='')
+url_encoded_taskid = urllib.parse.quote(task_id, safe="")
 print(f"TASKID: [{task_id}]")
-print('''
+print("""
 Re-query tasks
-==============''')
+==============""")
 tasks = druid.get_tasks(
     url=f"{protocol}://{druid_cluster_name}-coordinator-default-0.{druid_cluster_name}-coordinator-default.{namespace}.svc.cluster.local:{coordinator_port}/druid/indexer/v1/tasks",
 )
 new_task_count = len(json.loads(tasks))
-print(f'new tasks: {new_task_count}')
-print(f'assert {new_task_count} == {task_count+1}')
+print(f"new tasks: {new_task_count}")
+print(f"assert {new_task_count} == {task_count+1}")
 assert new_task_count == task_count + 1
 
-print('''
+print("""
 Wait for ingestion task to succeed
-======================================''')
+======================================""")
 job_finished = False
 while not job_finished:
     time.sleep(5)
@@ -123,29 +131,33 @@ def query_datasource(self, url, sql, expected, iterations):
     )
     task_status = json.loads(task)["status"]["statusCode"]
     print(f"Current task status: [{task_status}]")
-    assert task_status == "RUNNING" or task_status == "SUCCESS", f"Taskstatus not running or succeeeded: {task_status}"
+    assert (
+        task_status == "RUNNING" or task_status == "SUCCESS"
+    ), f"Taskstatus not running or succeeeded: {task_status}"
     job_finished = task_status == "SUCCESS"
 
-print('''
+print("""
 Wait for broker to indicate all segments are fully online
-======================================''')
+======================================""")
 broker_ready = False
 while not broker_ready:
     time.sleep(2)
-    broker_ready_rc = druid.check_rc(f"{protocol}://{druid_cluster_name}-broker-default-0.{druid_cluster_name}-broker-default.{namespace}.svc.cluster.local:{broker_port}/druid/broker/v1/readiness")
+    broker_ready_rc = druid.check_rc(
+        f"{protocol}://{druid_cluster_name}-broker-default-0.{druid_cluster_name}-broker-default.{namespace}.svc.cluster.local:{broker_port}/druid/broker/v1/readiness"
+    )
     broker_ready = broker_ready_rc == 200
     print(f"Broker respondend with [{broker_ready_rc}] to readiness check")
 
-print('''
+print("""
 Datasource SQL
-==============''')
+==============""")
 sample_data_size = 39244
 result = druid.query_datasource(
     url=f"{protocol}://{druid_cluster_name}-broker-default-0.{druid_cluster_name}-broker-default.{namespace}.svc.cluster.local:{broker_port}/druid/v2/sql",
-    sql={"query": "select count(*) as c from \"wikipedia-2015-09-12\""},
+    sql={"query": 'select count(*) as c from "wikipedia-2015-09-12"'},
     expected=sample_data_size,
-    iterations=12
+    iterations=12,
 )
-print(f'results: {result}')
-print(f'assert {sample_data_size} == {result}')
+print(f"results: {result}")
+print(f"assert {sample_data_size} == {result}")
 assert sample_data_size == result