Implements apache/operserverless#172

.github/cisetup.sh

@@ -31,7 +31,7 @@ curl -sL $URL | tar tzvf - -C ~/.local/bin
 YQ_VER=v4.27.2
 YQ_BIN=yq_linux_amd64
 sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VER}/${YQ_BIN} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
-MC_VER=RELEASE.2023-03-23T20-03-04Z
+MC_VER=RELEASE.2025-05-21T01-59-54Z
 sudo wget https://dl.min.io/client/mc/release/linux-${ARCH}/archive/mc.${MC_VER} -O /usr/bin/mc && sudo chmod +x /usr/bin/mc
 #URL="https://dl.k8s.io/release/$VER/bin/linux/$ARCH/kubectl"
 #curl -sSL "$URL" | sudo tee /usr/local/bin/kubectl && sudo chmod +x /usr/bin/kubectl

Dockerfile

@@ -70,6 +70,7 @@ ADD --chown=nuvolaris:nuvolaris deploy/milvus-operator /home/nuvolaris/deploy/mi
 ADD --chown=nuvolaris:nuvolaris deploy/milvus /home/nuvolaris/deploy/milvus
 ADD --chown=nuvolaris:nuvolaris deploy/milvus-slim /home/nuvolaris/deploy/milvus-slim
 ADD --chown=nuvolaris:nuvolaris deploy/registry /home/nuvolaris/deploy/registry
+ADD --chown=nuvolaris:nuvolaris deploy/seaweedfs /home/nuvolaris/deploy/seaweedfs
 ADD --chown=nuvolaris:nuvolaris quota.sh /home/nuvolaris/
 
 #------------------------------------------------------------------------------

deploy/nuvolaris-permissions/whisk-crd.yaml

@@ -149,7 +149,10 @@ spec:
                       type: boolean
                     registry:
                       description: deploys a private registry to load on the fly generated action runtimes (false by default)
-                      type: boolean                                              
+                      type: boolean
+                    seaweedfs:
+                      description: deploys an S3 compatible layer using a standalone deployment of seaweedfs
+                      type: boolean                                                                    
                   required:
                     - openwhisk
                     - couchdb
@@ -751,7 +754,10 @@ spec:
                             type: integer
                           ledgers:
                             description: pulsar bookie ledgers instance volume size
-                            type: integer                                                                                                                                     
+                            type: integer 
+                          bucket:
+                            description: used to setup a quota on the S3 bucket when running under seaweedfs (default to 10240MB)
+                            type: integer                                                                                                                                                                 
                     replicas:
                       description: number of total milvus replicas. Defaulted to 1
                       type: integer
@@ -835,7 +841,50 @@ spec:
                   - mode
                   - volume-size
                   - auth
-                  - hostname                                         
+                  - hostname
+                seaweedfs:
+                  description: used to configure the internal saweedfs data storage service
+                  type: object
+                  properties:
+                    volume-size:
+                      description: volume size in GB, default to 60GB
+                      type: integer
+                    default-bucket-quota:
+                      description: default bucket quota, default to 1024MB
+                      type: integer
+                    nuvolaris:
+                      description: used to configure the MINIO nuvolaris user used for non administrative purposes
+                      type: object
+                      properties:
+                        user:
+                          type: string
+                        password:
+                          type: string
+                      required:
+                      - user
+                      - password
+                    ingress:
+                      description: configuration option for global minio ingresses exposure
+                      type: object
+                      properties:
+                        s3-enabled:
+                          description: boolean flag to activate SEAWEEDFS S3 compatible ingress. Default to false
+                          type: boolean
+                        s3-hostname:
+                          description: ingress hostname to be used if true (normally s3.<apihost>)
+                          type: string
+                        console-enabled:
+                          description: boolean flag to expose SEAWEEDFS Filer UI
+                          type: boolean
+                        console-hostname:
+                          description: ingress hostname to be used (nora=mally filer.<apihost>)
+                          type: string
+                      required:
+                      - s3-enabled
+                      - console-enabled                           
+                  required:
+                  - volume-size
+                  - nuvolaris                                                           
             status:
               x-kubernetes-preserve-unknown-fields: true
               # type: object

deploy/redis/redis-set.yaml

@@ -37,7 +37,7 @@ spec:
       restartPolicy: Always    
       containers:
       - name: redis
-        image: bitnami/valkey:7.2.5
+        image: bitnamisecure/valkey:latest
         command: ["/bin/sh","-c","redis-server /redis-master/redis.conf"]
         env:
         - name: MASTER

deploy/seaweedfs/seaweedfs-pvc.yaml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: seaweedfs-pvc
+  namespace: nuvolaris
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 30Gi

deploy/seaweedfs/seaweedfs-sts.yaml

@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: seaweedfs
+  namespace: nuvolaris
+spec:
+  serviceName: "seaweedfs"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: seaweedfs
+  template:
+    metadata:
+      labels:
+        app: seaweedfs
+    spec:
+      containers:
+        - name: seaweedfs
+          image: chrislusf/seaweedfs:3.97
+          ports:
+            - containerPort: 9000 # S3
+            - containerPort: 9090 # Filer
+            - containerPort: 9333 # Master
+          command: ["weed"]
+          args:
+            - "server"
+            - "-s3"
+            - "-s3.port=9000"
+            - "-filer"
+            - "-filer.port=9090"
+            - "-master.port=9333"
+            - "-dir=/data"
+            - "-volume.max=0"
+            - "-master.volumeSizeLimitMB=1024"
+          volumeMounts:
+            - name: seaweedfs-data
+              mountPath: /data
+      volumes:
+        - name: seaweedfs-data
+          persistentVolumeClaim:
+            claimName: seaweedfs-pvc

deploy/seaweedfs/seaweedfs-svc.yaml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: seaweedfs
+  namespace: nuvolaris
+spec:
+  type: NodePort
+  ports:
+    - name: s3-api
+      port: 9000
+      targetPort: 9000
+      nodePort: 32090
+      protocol: TCP
+    - name: filer
+      port: 9090
+      targetPort: 9090
+      nodePort: 32091
+      protocol: TCP
+    - name: master
+      port: 9333
+      targetPort: 9333
+      nodePort: 32093
+      protocol: TCP
+  selector:
+    app: seaweedfs

nuvolaris/kopf_util.py

@@ -73,6 +73,14 @@ def check_minio_ingresses(response: dict, item: dict):
         if(item['new']):
             response["minio-ingresses"]="update"
 
+def check_seaweedfs_ingresses(response: dict, item: dict):
+    """
+    Forces an update of seaweed-ingresses if needed
+    """
+    if(item['path']=='spec.seaweedfs.ingress.s3-enabled' or item['path']=='spec.seaweedfs.ingress.console-enabled'):
+        if(item['new']):
+            response["seaweedfs-ingresses"]="update"            
+
 def check_registry_ingresses(response: dict, item: dict):
     """
     Forces an update of registry-ingress if needed
@@ -100,9 +108,11 @@ def evaluate_differences(response: dict, differences: list):
         check_component(response, d,"spec.components.etcd","etcd")
         check_component(response, d,"spec.components.milvus","milvus")
         check_component(response, d,"spec.components.registry","registry")
+        check_component(response, d,"spec.components.seaweedfs","seaweedfs")
         openwhisk(response, d)           
         endpoint(response, d)
         check_minio_ingresses(response, d)
+        check_seaweedfs_ingresses(response, d)
 
 def detect_component_changes(kopf_diff):
     """

nuvolaris/main.py

@@ -42,6 +42,7 @@
 import nuvolaris.etcd as etcd
 import nuvolaris.milvus_standalone as milvus
 import nuvolaris.registry_deploy as registry
+import nuvolaris.seaweedfs_deploy as seaweedfs
 
 @kopf.on.startup()
 def configure(settings: kopf.OperatorSettings, **_):
@@ -84,6 +85,10 @@ def whisk_create(spec, name, **kwargs):
         "etcd":"?" #Etcdd configuration
     }
 
+    if cfg.get('components.minio') and cfg.get('components.seaweedfs'):
+        state['controller']= "NotValid"
+        raise kopf.PermanentError("Storage support for MINIO and SEAWEEDFS could not be activated simultaneously.")    
+
     runtime = cfg.get('nuvolaris.kube')
     logging.info(f"kubernetes engine in use={runtime}")
 
@@ -93,7 +98,7 @@ def whisk_create(spec, name, **kwargs):
             state['preloader']= "on"
             logging.info(msg)
         except:
-            logging.exception("could not create runtime preloader batach")
+            logging.exception("could not create runtime preloader batch")
             state['preloader']= "error"
     else:
         state['preloader']= "off"   
@@ -165,6 +170,13 @@ def whisk_create(spec, name, **kwargs):
     else:
         state['minio'] = "off"
 
+    if cfg.get('components.seaweedfs'):
+        msg = seaweedfs.create(owner)
+        logging.info(msg)
+        state['seaweedfs'] = "on"
+    else:
+        state['seaweedfs'] = "off"         
+
     if cfg.get('components.static'):
         msg = static.create(owner)
         logging.info(msg)
@@ -332,6 +344,10 @@ def whisk_delete(spec, **kwargs):
         msg = minio.delete()
         logging.info(msg)
 
+    if cfg.get("components.seaweedfs"):
+        msg = seaweedfs.delete()
+        logging.info(msg)        
+
     if cfg.get('components.postgres'):
         msg = postgres.delete()
         logging.info(msg)

nuvolaris/milvus_standalone.py

@@ -29,6 +29,7 @@
 from nuvolaris.milvus_admin_client import MilvusAdminClient
 from nuvolaris.user_config import UserConfig
 from nuvolaris.user_metadata import UserMetadata
+from nuvolaris.seaweedfs_util import SeaweedfsClient
 
 
 def patchEntries(data: dict):
@@ -105,7 +106,7 @@ def create(owner=None):
             r"{.items[?(@.metadata.labels.app\.kubernetes\.io\/instance == 'nuvolaris-milvus')].metadata.name}")
 
         milvus_api_host = cfg.get("milvus.host", "MILVUS_API_HOST", "nuvolaris-milvus")
-        milvus_api_port = cfg.get("milvus.host", "MILVUS_API_PORT", "19530")
+        milvus_api_port = cfg.get("milvus.port", "MILVUS_API_PORT", "19530")
 
         logging.info("*** waiting for milvus api to be available")
         util.wait_for_http(f"http://{milvus_api_host}:{milvus_api_port}", up_statuses=[200,401], timeout=30)
@@ -117,14 +118,11 @@ def create(owner=None):
     return res
 
 
-def create_milvus_accounts(data: dict):
+def create_minio_milvus_account(data: dict):
     """"
-    Creates technical accounts for ETCD and MINIO
+    Creates technical accounts for MINIO
     """
     try:
-        # currently we use the ETCD root password, so we skip the ETCD user creation.
-        # res = util.check(etcd.create_etcd_user(data['milvus_etcd_username'],data['milvus_etcd_password'],data['milvus_etcd_prefix']),"create_etcd_milvus_user",True)
-
         minioClient = mutil.MinioClient()
         bucket_policy_names = []
         bucket_policy_names.append(f"{data['milvus_bucket_name']}/*")
@@ -135,8 +133,33 @@ def create_milvus_accounts(data: dict):
         return util.check(minioClient.assign_rw_bucket_policy_to_user(data["milvus_s3_username"], bucket_policy_names),
                           "assign_milvus_s3_bucket_policy", res)
     except Exception as ex:
-        logging.error("Could not create milvus ETCD and MINIO accounts", ex)
-        return False
+        logging.error("Could not create milvus MINIO accounts", ex)
+        return False 
+
+def create_seaweedfs_milvus_account(data: dict):
+    """"
+    Creates technical accounts for SEAWEEDFS
+    """
+    try:
+        seaweedfsClient = SeaweedfsClient()
+        res = util.check(seaweedfsClient.make_bucket(data["milvus_bucket_name"],data["milvus_bucket_quota"]),"make_milvus_bucket",True)
+        return util.check(seaweedfsClient.add_user(data["milvus_s3_username"],data["milvus_s3_username"],data["milvus_s3_password"],data["milvus_bucket_name"]),"add_milvus_user",res)
+    except Exception as ex:
+        logging.error("Could not create milvus SEAWEEDFS accounts", ex)
+        return False        
+
+def create_milvus_accounts(data: dict):
+    """"
+    Creates technical accounts for ETCD and MINIO
+    """
+    # currently we use the ETCD root password, so we skip the ETCD user creation.
+    # res = util.check(etcd.create_etcd_user(data['milvus_etcd_username'],data['milvus_etcd_password'],data['milvus_etcd_prefix']),"create_etcd_milvus_user",True)
+
+    if cfg.get('components.minio'):
+        return create_minio_milvus_account(data)
+
+    if cfg.get('components.seaweedfs'):
+        return create_seaweedfs_milvus_account(data)
 
 
 def create_default_milvus_database(data):