add precommit setup and run across files, fix dockerfile again

Author: jzvolensky

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+    -   id: check-added-large-files
+    -   id: check-merge-conflict
+    -   id: debug-statements
+    -   id: check-docstring-first
+-   repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v2.0.4
+    hooks:
+    -   id: autopep8
+        args: [--in-place, --aggressive, --aggressive, --max-line-length=88]
+-   repo: https://github.com/psf/black
+    rev: 23.12.1
+    hooks:
+    -   id: black
+        language_version: python3
+        args: [--line-length=88]
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+        args: [--profile=black, --line-length=88]
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+    -   id: flake8
+        args: [--max-line-length=88, --ignore=E203,W503]
+        exclude: ^(archive/|docker/dummy/)

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to HyDroForM (WIP)
+# Contributing to HyDroForM
 
 First off, thanks for taking the time to contribute! ❤️
 
@@ -39,7 +39,7 @@ We will then take care of the issue as soon as possible.
 
 ## I Want To Contribute
 
-> ### Legal Notice 
+> ### Legal Notice
 > When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
 
 ### Reporting Bugs
@@ -101,7 +101,7 @@ Enhancement suggestions are tracked as [GitHub issues](/issues).
 - Use a **clear and descriptive title** for the issue to identify the suggestion.
 - Provide a **step-by-step description of the suggested enhancement** in as many details as possible.
 - **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you.
-- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. 
+- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
 - **Explain why this enhancement would be useful** to most CONTRIBUTING.md users. You may also want to point out the other projects that solved it better and which could serve as inspiration.
 
 

OSCAR/examples/old/oscar.py

@@ -1,16 +1,17 @@
 import argparse
 import json
 import os
-import sys
 import subprocess
-from minio import Minio
-from oscar_python.client import Client
-import oscar_python._utils as utils
+import sys
 import tarfile
 import uuid
-import requests
+
 import openeo
-import subprocess
+import oscar_python._utils as utils
+import requests
+from minio import Minio
+from oscar_python.client import Client
+
 
 def parse_arguments():
     # parser = argparse.ArgumentParser()
@@ -38,20 +39,24 @@ def parse_arguments():
     # service_config = variables['service_config']  # os.path.abspath(
     # output = variables['output']
 
-    conn = openeo.connect("https://openeo.intertwin.fedcloud.eu/1.1.0").authenticate_oidc()
+    conn = openeo.connect(
+        "https://openeo.intertwin.fedcloud.eu/1.1.0"
+    ).authenticate_oidc()
 
     TOKEN = "/home/jzvolensky/.local/share/openeo-python-client/refresh-tokens.json"
     with open(TOKEN) as f:
         token_data = json.load(f)
-        refresh_token = token_data["https://aai.egi.eu/auth/realms/egi"]["openeo-platform-default-client"]["refresh_token"].strip()
+        refresh_token = token_data["https://aai.egi.eu/auth/realms/egi"][
+            "openeo-platform-default-client"
+        ]["refresh_token"].strip()
     print(refresh_token)
 
     token = get_access_token(refresh_token)
 
     endpoint = "https://oscar-grnet.intertwin.fedcloud.eu"
     user = ""
     password = ""
-    token = f'{token}'
+    token = f"{token}"
     service = "hydroform"
     service_config = "oscar_services/hydroform_oscar_svc.yaml"
     output = "output"
@@ -67,11 +72,10 @@ def parse_arguments():
 
 def get_access_token(refresh_token):
     url = "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token"
-    data = f"grant_type=refresh_token&refresh_token={refresh_token}&client_id=openeo-platform-default-client&scope=openid%20email%20offline_access%20eduperson_scoped_affiliation%20eduperson_entitlement"
+    data = f"grant_type=refresh_token&refresh_token={
+        refresh_token}&client_id=openeo-platform-default-client&scope=openid%20email%20offline_access%20eduperson_scoped_affiliation%20eduperson_entitlement"
     result = subprocess.run(
-    ["curl", "-X", "POST", url, "-d", data],
-    stdout=subprocess.PIPE,
-    text=True
+        ["curl", "-X", "POST", url, "-d", data], stdout=subprocess.PIPE, text=True
     )
     if result.returncode == 0:
         token_info = json.loads(result.stdout)
@@ -80,22 +84,27 @@ def get_access_token(refresh_token):
         print(f"Failed to get access token: {result.returncode}")
 
     return token_info["access_token"]
-    
+
+
 def check_oscar_connection():
     # Check the service or create it
     print("Checking OSCAR connection status")
     if user and password:
-        options_basic_auth = {'cluster_id': 'cluster-id',
-                              'endpoint': endpoint,
-                              'user': user,
-                              'password': password,
-                              'ssl': 'True'}
+        options_basic_auth = {
+            "cluster_id": "cluster-id",
+            "endpoint": endpoint,
+            "user": user,
+            "password": password,
+            "ssl": "True",
+        }
         print("Using credentials user/password")
     elif token:
-        options_basic_auth = {'cluster_id': 'cluster-id',
-                              'endpoint': endpoint,
-                              'oidc_token': token,
-                              'ssl': 'True'}
+        options_basic_auth = {
+            "cluster_id": "cluster-id",
+            "endpoint": endpoint,
+            "oidc_token": token,
+            "ssl": "True",
+        }
         print("Using credentials token")
     else:
         print("Introduce the credentials user/password or token")
@@ -133,7 +142,9 @@ def check_service(client, service, service_config):
             exit(1)
     try:
         service_info = client.get_service(service)
-        minio_info = json.loads(service_info.text)["storage_providers"]["minio"]["default"]
+        minio_info = json.loads(service_info.text)["storage_providers"]["minio"][
+            "default"
+        ]
         input_info = json.loads(service_info.text)["input"][0]
         output_info = json.loads(service_info.text)["output"][0]
         print("OSCAR Service " + service + " created")
@@ -147,9 +158,11 @@ def check_service(client, service, service_config):
 def connect_minio(minio_info):
     # Create client with access and secret key.
     print("Creating connection with MinIO")
-    client = Minio(minio_info["endpoint"].split("//")[1],
-                   minio_info["access_key"],
-                   minio_info["secret_key"])
+    client = Minio(
+        minio_info["endpoint"].split("//")[1],
+        minio_info["access_key"],
+        minio_info["secret_key"],
+    )
     return client
 
 
@@ -159,7 +172,7 @@ def upload_file_minio(client, input_info, input_file):
     random = uuid.uuid4().hex + "_" + input_file.split("/")[-1]
     client.fput_object(
         input_info["path"].split("/")[0],
-        '/'.join(input_info["path"].split("/")[1:]) + "/" + random,
+        "/".join(input_info["path"].split("/")[1:]) + "/" + random,
         input_file,
     )
     return random.split("_")[0]
@@ -170,7 +183,7 @@ def wait_output_and_download(client, output_info, output):
     print("Waiting the output")
     with client.listen_bucket_notification(
         output_info["path"].split("/")[0],
-        prefix='/'.join(output_info["path"].split("/")[1:]),
+        prefix="/".join(output_info["path"].split("/")[1:]),
         events=["s3:ObjectCreated:*", "s3:ObjectRemoved:*"],
     ) as events:
         for event in events:
@@ -179,9 +192,11 @@ def wait_output_and_download(client, output_info, output):
             break
     # Download the file
     print("Downloading the file")
-    client.fget_object(output_info["path"].split("/")[0], 
-                       outputfile,
-                       output + "/" + outputfile.split("/")[-1])
+    client.fget_object(
+        output_info["path"].split("/")[0],
+        outputfile,
+        output + "/" + outputfile.split("/")[-1],
+    )
     return output + "/" + outputfile.split("/")[-1]
 
 
@@ -197,31 +212,45 @@ def compress(filename):
 
 def extract(output_file):
     print("Decompressing output")
-    with tarfile.open(output_file, 'r') as tar:
+    with tarfile.open(output_file, "r") as tar:
         for member in tar.getmembers():
             tar.extract(member, path=output)
 
 
 def run_service(client, service, token, output):
-    print(f'Running {service} service')
+    print(f"Running {service} service")
     response = None
     try:
         data = {
             "Records": [
                 {
                     "requestParameters": {
                         "principalId": "uid",
-                        "sourceIPAddress": "ip"
+                        "sourceIPAddress": "ip",
                     },
                 }
             ]
         }
-        json_data = json.dumps(data).encode('utf-8')
+        json_data = json.dumps(data).encode("utf-8")
         if token:
             headers = utils.get_headers_with_token(token)
-            response = requests.request("post", endpoint + "/job/" + service, headers=headers, verify=client.ssl, data=json_data, timeout=1500)
+            response = requests.request(
+                "post",
+                endpoint + "/job/" + service,
+                headers=headers,
+                verify=client.ssl,
+                data=json_data,
+                timeout=1500,
+            )
         elif user and password:
-            response = requests.request("post", endpoint + "/job/" + service, auth=(user, password), verify=client.ssl, data=json_data, timeout=1500)
+            response = requests.request(
+                "post",
+                endpoint + "/job/" + service,
+                auth=(user, password),
+                verify=client.ssl,
+                data=json_data,
+                timeout=1500,
+            )
         else:
             raise ValueError("Either token or user/password must be provided")
     except Exception as err:
@@ -236,5 +265,7 @@ def run_service(client, service, token, output):
     minio_info, input_info, output_info = check_service(client, service, service_config)
     print(minio_info, input_info, output_info)
     response = run_service(client, service, token, output)
-    output_file = wait_output_and_download(connect_minio(minio_info), output_info, output)
+    output_file = wait_output_and_download(
+        connect_minio(minio_info), output_info, output
+    )
     extract(output_file)

OSCAR/examples/old/oscar_services/hydroform_oscar_svc.yaml

@@ -13,4 +13,4 @@ functions:
         path: hydroform/in
       output:
       - storage_provider: minio
-        path: hydroform/out
+        path: hydroform/out

OSCAR/examples/old/oscar_services/script.sh

@@ -16,7 +16,7 @@ echo "catalog: $catalog"
 hydromt build wflow model \
 -r "$region" \
 -d "$catalog" \
--i "$setupconfig" -vvv 
+-i "$setupconfig" -vvv
 echo "Finished running hydromt build wflow model"
 
 echo "Wrapping output files"
@@ -27,4 +27,4 @@ echo "Finished wrapping output files"
 tar -czvf output.tar.gz model/stac
 mv output.tar.gz $TMP_OUTPUT_DIR/output.tar.gz
 
-echo "End of script"
+echo "End of script"