diff --git a/.github/cisetup.sh b/.github/cisetup.sh index 8f7b7d7..80a073b 100644 --- a/.github/cisetup.sh +++ b/.github/cisetup.sh @@ -31,7 +31,7 @@ curl -sL $URL | tar tzvf - -C ~/.local/bin YQ_VER=v4.27.2 YQ_BIN=yq_linux_amd64 sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VER}/${YQ_BIN} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq -MC_VER=RELEASE.2023-03-23T20-03-04Z +MC_VER=RELEASE.2025-05-21T01-59-54Z sudo wget https://dl.min.io/client/mc/release/linux-${ARCH}/archive/mc.${MC_VER} -O /usr/bin/mc && sudo chmod +x /usr/bin/mc #URL="https://dl.k8s.io/release/$VER/bin/linux/$ARCH/kubectl" #curl -sSL "$URL" | sudo tee /usr/local/bin/kubectl && sudo chmod +x /usr/bin/kubectl diff --git a/Dockerfile b/Dockerfile index e460f0d..99d3441 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,6 +70,7 @@ ADD --chown=nuvolaris:nuvolaris deploy/milvus-operator /home/nuvolaris/deploy/mi ADD --chown=nuvolaris:nuvolaris deploy/milvus /home/nuvolaris/deploy/milvus ADD --chown=nuvolaris:nuvolaris deploy/milvus-slim /home/nuvolaris/deploy/milvus-slim ADD --chown=nuvolaris:nuvolaris deploy/registry /home/nuvolaris/deploy/registry +ADD --chown=nuvolaris:nuvolaris deploy/seaweedfs /home/nuvolaris/deploy/seaweedfs ADD --chown=nuvolaris:nuvolaris quota.sh /home/nuvolaris/ #------------------------------------------------------------------------------ diff --git a/deploy/nuvolaris-permissions/whisk-crd.yaml b/deploy/nuvolaris-permissions/whisk-crd.yaml index dff969c..78f3f03 100644 --- a/deploy/nuvolaris-permissions/whisk-crd.yaml +++ b/deploy/nuvolaris-permissions/whisk-crd.yaml @@ -149,7 +149,10 @@ spec: type: boolean registry: description: deploys a private registry to load on the fly generated action runtimes (false by default) - type: boolean + type: boolean + seaweedfs: + description: deploys an S3 compatible layer using a standalone deployment of seaweedfs + type: boolean required: - openwhisk - couchdb @@ -751,7 +754,10 @@ spec: type: integer ledgers: description: pulsar bookie ledgers instance volume size - type: integer + type: integer + bucket: + description: used to setup a quota on the S3 bucket when running under seaweedfs (default to 10240MB) + type: integer replicas: description: number of total milvus replicas. Defaulted to 1 type: integer @@ -835,7 +841,50 @@ spec: - mode - volume-size - auth - - hostname + - hostname + seaweedfs: + description: used to configure the internal saweedfs data storage service + type: object + properties: + volume-size: + description: volume size in GB, default to 60GB + type: integer + default-bucket-quota: + description: default bucket quota, default to 1024MB + type: integer + nuvolaris: + description: used to configure the MINIO nuvolaris user used for non administrative purposes + type: object + properties: + user: + type: string + password: + type: string + required: + - user + - password + ingress: + description: configuration option for global minio ingresses exposure + type: object + properties: + s3-enabled: + description: boolean flag to activate SEAWEEDFS S3 compatible ingress. Default to false + type: boolean + s3-hostname: + description: ingress hostname to be used if true (normally s3.) + type: string + console-enabled: + description: boolean flag to expose SEAWEEDFS Filer UI + type: boolean + console-hostname: + description: ingress hostname to be used (nora=mally filer.) + type: string + required: + - s3-enabled + - console-enabled + required: + - volume-size + - nuvolaris status: x-kubernetes-preserve-unknown-fields: true # type: object diff --git a/deploy/redis/redis-set.yaml b/deploy/redis/redis-set.yaml index 8deaa6d..a73b88e 100644 --- a/deploy/redis/redis-set.yaml +++ b/deploy/redis/redis-set.yaml @@ -37,7 +37,7 @@ spec: restartPolicy: Always containers: - name: redis - image: bitnami/valkey:7.2.5 + image: bitnamisecure/valkey:latest command: ["/bin/sh","-c","redis-server /redis-master/redis.conf"] env: - name: MASTER diff --git a/deploy/seaweedfs/seaweedfs-pvc.yaml b/deploy/seaweedfs/seaweedfs-pvc.yaml new file mode 100644 index 0000000..3a27d76 --- /dev/null +++ b/deploy/seaweedfs/seaweedfs-pvc.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: seaweedfs-pvc + namespace: nuvolaris +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi \ No newline at end of file diff --git a/deploy/seaweedfs/seaweedfs-sts.yaml b/deploy/seaweedfs/seaweedfs-sts.yaml new file mode 100644 index 0000000..a73236a --- /dev/null +++ b/deploy/seaweedfs/seaweedfs-sts.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: seaweedfs + namespace: nuvolaris +spec: + serviceName: "seaweedfs" + replicas: 1 + selector: + matchLabels: + app: seaweedfs + template: + metadata: + labels: + app: seaweedfs + spec: + containers: + - name: seaweedfs + image: chrislusf/seaweedfs:3.97 + ports: + - containerPort: 9000 # S3 + - containerPort: 9090 # Filer + - containerPort: 9333 # Master + command: ["weed"] + args: + - "server" + - "-s3" + - "-s3.port=9000" + - "-filer" + - "-filer.port=9090" + - "-master.port=9333" + - "-dir=/data" + - "-volume.max=0" + - "-master.volumeSizeLimitMB=1024" + volumeMounts: + - name: seaweedfs-data + mountPath: /data + volumes: + - name: seaweedfs-data + persistentVolumeClaim: + claimName: seaweedfs-pvc diff --git a/deploy/seaweedfs/seaweedfs-svc.yaml b/deploy/seaweedfs/seaweedfs-svc.yaml new file mode 100644 index 0000000..3c10dfb --- /dev/null +++ b/deploy/seaweedfs/seaweedfs-svc.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +apiVersion: v1 +kind: Service +metadata: + name: seaweedfs + namespace: nuvolaris +spec: + type: NodePort + ports: + - name: s3-api + port: 9000 + targetPort: 9000 + nodePort: 32090 + protocol: TCP + - name: filer + port: 9090 + targetPort: 9090 + nodePort: 32091 + protocol: TCP + - name: master + port: 9333 + targetPort: 9333 + nodePort: 32093 + protocol: TCP + selector: + app: seaweedfs \ No newline at end of file diff --git a/nuvolaris/kopf_util.py b/nuvolaris/kopf_util.py index 72b619e..cce7972 100644 --- a/nuvolaris/kopf_util.py +++ b/nuvolaris/kopf_util.py @@ -73,6 +73,14 @@ def check_minio_ingresses(response: dict, item: dict): if(item['new']): response["minio-ingresses"]="update" +def check_seaweedfs_ingresses(response: dict, item: dict): + """ + Forces an update of seaweed-ingresses if needed + """ + if(item['path']=='spec.seaweedfs.ingress.s3-enabled' or item['path']=='spec.seaweedfs.ingress.console-enabled'): + if(item['new']): + response["seaweedfs-ingresses"]="update" + def check_registry_ingresses(response: dict, item: dict): """ Forces an update of registry-ingress if needed @@ -100,9 +108,11 @@ def evaluate_differences(response: dict, differences: list): check_component(response, d,"spec.components.etcd","etcd") check_component(response, d,"spec.components.milvus","milvus") check_component(response, d,"spec.components.registry","registry") + check_component(response, d,"spec.components.seaweedfs","seaweedfs") openwhisk(response, d) endpoint(response, d) check_minio_ingresses(response, d) + check_seaweedfs_ingresses(response, d) def detect_component_changes(kopf_diff): """ diff --git a/nuvolaris/main.py b/nuvolaris/main.py index bae0217..f5b797d 100644 --- a/nuvolaris/main.py +++ b/nuvolaris/main.py @@ -42,6 +42,7 @@ import nuvolaris.etcd as etcd import nuvolaris.milvus_standalone as milvus import nuvolaris.registry_deploy as registry +import nuvolaris.seaweedfs_deploy as seaweedfs @kopf.on.startup() def configure(settings: kopf.OperatorSettings, **_): @@ -84,6 +85,10 @@ def whisk_create(spec, name, **kwargs): "etcd":"?" #Etcdd configuration } + if cfg.get('components.minio') and cfg.get('components.seaweedfs'): + state['controller']= "NotValid" + raise kopf.PermanentError("Storage support for MINIO and SEAWEEDFS could not be activated simultaneously.") + runtime = cfg.get('nuvolaris.kube') logging.info(f"kubernetes engine in use={runtime}") @@ -93,7 +98,7 @@ def whisk_create(spec, name, **kwargs): state['preloader']= "on" logging.info(msg) except: - logging.exception("could not create runtime preloader batach") + logging.exception("could not create runtime preloader batch") state['preloader']= "error" else: state['preloader']= "off" @@ -165,6 +170,13 @@ def whisk_create(spec, name, **kwargs): else: state['minio'] = "off" + if cfg.get('components.seaweedfs'): + msg = seaweedfs.create(owner) + logging.info(msg) + state['seaweedfs'] = "on" + else: + state['seaweedfs'] = "off" + if cfg.get('components.static'): msg = static.create(owner) logging.info(msg) @@ -332,6 +344,10 @@ def whisk_delete(spec, **kwargs): msg = minio.delete() logging.info(msg) + if cfg.get("components.seaweedfs"): + msg = seaweedfs.delete() + logging.info(msg) + if cfg.get('components.postgres'): msg = postgres.delete() logging.info(msg) diff --git a/nuvolaris/milvus_standalone.py b/nuvolaris/milvus_standalone.py index 0dc8313..fe702d2 100644 --- a/nuvolaris/milvus_standalone.py +++ b/nuvolaris/milvus_standalone.py @@ -29,6 +29,7 @@ from nuvolaris.milvus_admin_client import MilvusAdminClient from nuvolaris.user_config import UserConfig from nuvolaris.user_metadata import UserMetadata +from nuvolaris.seaweedfs_util import SeaweedfsClient def patchEntries(data: dict): @@ -105,7 +106,7 @@ def create(owner=None): r"{.items[?(@.metadata.labels.app\.kubernetes\.io\/instance == 'nuvolaris-milvus')].metadata.name}") milvus_api_host = cfg.get("milvus.host", "MILVUS_API_HOST", "nuvolaris-milvus") - milvus_api_port = cfg.get("milvus.host", "MILVUS_API_PORT", "19530") + milvus_api_port = cfg.get("milvus.port", "MILVUS_API_PORT", "19530") logging.info("*** waiting for milvus api to be available") util.wait_for_http(f"http://{milvus_api_host}:{milvus_api_port}", up_statuses=[200,401], timeout=30) @@ -117,14 +118,11 @@ def create(owner=None): return res -def create_milvus_accounts(data: dict): +def create_minio_milvus_account(data: dict): """" - Creates technical accounts for ETCD and MINIO + Creates technical accounts for MINIO """ try: - # currently we use the ETCD root password, so we skip the ETCD user creation. - # res = util.check(etcd.create_etcd_user(data['milvus_etcd_username'],data['milvus_etcd_password'],data['milvus_etcd_prefix']),"create_etcd_milvus_user",True) - minioClient = mutil.MinioClient() bucket_policy_names = [] bucket_policy_names.append(f"{data['milvus_bucket_name']}/*") @@ -135,8 +133,33 @@ def create_milvus_accounts(data: dict): return util.check(minioClient.assign_rw_bucket_policy_to_user(data["milvus_s3_username"], bucket_policy_names), "assign_milvus_s3_bucket_policy", res) except Exception as ex: - logging.error("Could not create milvus ETCD and MINIO accounts", ex) - return False + logging.error("Could not create milvus MINIO accounts", ex) + return False + +def create_seaweedfs_milvus_account(data: dict): + """" + Creates technical accounts for SEAWEEDFS + """ + try: + seaweedfsClient = SeaweedfsClient() + res = util.check(seaweedfsClient.make_bucket(data["milvus_bucket_name"],data["milvus_bucket_quota"]),"make_milvus_bucket",True) + return util.check(seaweedfsClient.add_user(data["milvus_s3_username"],data["milvus_s3_username"],data["milvus_s3_password"],data["milvus_bucket_name"]),"add_milvus_user",res) + except Exception as ex: + logging.error("Could not create milvus SEAWEEDFS accounts", ex) + return False + +def create_milvus_accounts(data: dict): + """" + Creates technical accounts for ETCD and MINIO + """ + # currently we use the ETCD root password, so we skip the ETCD user creation. + # res = util.check(etcd.create_etcd_user(data['milvus_etcd_username'],data['milvus_etcd_password'],data['milvus_etcd_prefix']),"create_etcd_milvus_user",True) + + if cfg.get('components.minio'): + return create_minio_milvus_account(data) + + if cfg.get('components.seaweedfs'): + return create_seaweedfs_milvus_account(data) def create_default_milvus_database(data): diff --git a/nuvolaris/patcher.py b/nuvolaris/patcher.py index 280b110..2ccdc3f 100644 --- a/nuvolaris/patcher.py +++ b/nuvolaris/patcher.py @@ -36,6 +36,7 @@ import nuvolaris.etcd as etcd import nuvolaris.milvus_standalone as milvus import nuvolaris.registry_deploy as registry +import nuvolaris.seaweedfs_deploy as seaweedfs def patch_preloader(owner: None): try: @@ -177,6 +178,9 @@ def patch(diff, status, owner=None, name=None): if "minio-ingresses" in what_to_do and what_to_do['minio-ingresses'] == "update": minio.patch_ingresses(status,what_to_do['minio-ingresses'], owner) + if "seaweedfs-ingresses" in what_to_do and what_to_do['seaweedfs-ingresses'] == "update": + seaweedfs.patch_ingresses(status,what_to_do['seaweedfs-ingresses'], owner) + if components_updated: operator_util.whisk_post_create(name) diff --git a/nuvolaris/seaweedfs_deploy.py b/nuvolaris/seaweedfs_deploy.py new file mode 100644 index 0000000..52a1c22 --- /dev/null +++ b/nuvolaris/seaweedfs_deploy.py @@ -0,0 +1,278 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import kopf, logging, time, os +import nuvolaris.kube as kube +import nuvolaris.kustomize as kus +import nuvolaris.config as cfg +import nuvolaris.util as util +import nuvolaris.openwhisk as openwhisk +import nuvolaris.seaweedfs_ingress as seaweedfs_ingress +import nuvolaris.operator_util as operator_util + +from nuvolaris.user_config import UserConfig +from nuvolaris.user_metadata import UserMetadata +from nuvolaris.seaweedfs_util import SeaweedfsClient + +def _get_seaweedfs_service(): + return util.get_service("{.items[?(@.spec.selector.app == 'seaweedfs')]}") + +def _add_seaweedfs_user_metadata(ucfg: UserConfig, user_metadata:UserMetadata): + """ + adds entries for seaweedfs connectivity S3_HOST, S3_PORT, S3_ACCESS_KEY, S3_SECRET_KEY + """ + + try: + seaweed_service = _get_seaweedfs_service() + if(seaweed_service): + seaweedfs_host = f"{seaweed_service['metadata']['name']}.{seaweed_service['metadata']['namespace']}.svc.cluster.local" + access_key = ucfg.get('namespace') + secret_key = ucfg.get("object-storage.password") + user_metadata.add_metadata("S3_PROVIDER","seaweedfs") + user_metadata.add_metadata("S3_HOST",seaweedfs_host) + user_metadata.add_metadata("S3_ACCESS_KEY",access_key) + user_metadata.add_metadata("S3_SECRET_KEY",secret_key) + + user_metadata.add_safely_from_cm("S3_API_URL", '{.metadata.annotations.s3_api_url}') + user_metadata.add_safely_from_cm("S3_CONSOLE_URL", '{.metadata.annotations.s3_console_url}') + + ports = list(seaweed_service['spec']['ports']) + for port in ports: + if(port['name']=='s3-api'): + user_metadata.add_metadata("S3_PORT",port['port']) + + return None + except Exception as e: + logging.error(f"failed to build SEAWEEDFS metadata for {ucfg.get('namespace')}: {e}") + return None + +def create(owner=None): + logging.info("*** configuring seaweedfs standalone") + + data = util.get_seaweedfs_config_data() + + tplp = ["pvc-attach.yaml"] + + if(data['affinity'] or data['tolerations']): + tplp.append("affinity-tolerance-sts-core-attach.yaml") + + kust = kus.patchTemplates("seaweedfs", tplp, data) + spec = kus.kustom_list("seaweedfs", kust, templates=[], data=data) + + if owner: + kopf.append_owner_reference(spec['items'], owner) + else: + cfg.put("state.seaweedfs.spec", spec) + + res = kube.apply(spec) + + # dynamically detect seaweedfs pod and wait for readiness + util.wait_for_pod_ready("{.items[?(@.metadata.labels.app == 'seaweedfs')].metadata.name}") + seaweedfs_ingress.create_seaweedfs_ingresses(data, owner) + + logging.info("*** waiting for seaweedfs filer api to be available") + util.wait_for_http(util.get_seaweedds_filer_host(), up_statuses=[200,401], timeout=30) + + create_seaweedfs_nuv_storage(data) + + logging.info("*** configured seaweedfs standalone") + return res + +def _annotate_nuv_metadata(data): + """ + annotate nuvolaris configmap with entries for minio connectivity S3_ENDPOINT, S3_PORT, S3_ACCESS_KEY, S3_SECRET_KEY + this is becasue MINIO + """ + try: + seaweed_service = _get_seaweedfs_service() + if(seaweed_service): + seaweed_host = f"{seaweed_service['metadata']['name']}.{seaweed_service['metadata']['namespace']}.svc.cluster.local" + access_key = data["seaweedfs_nuv_user"] + secret_key = data["seaweedfs_nuv_password"] + openwhisk.annotate(f"s3_host={seaweed_host}") + openwhisk.annotate(f"s3_access_key={access_key}") + openwhisk.annotate(f"s3_secret_key={secret_key}") + openwhisk.annotate("s3_provider=seaweedfs") + + ports = list(seaweed_service['spec']['ports']) + for port in ports: + if(port['name']=='s3-api'): + openwhisk.annotate(f"s3_port={port['port']}") + return None + except Exception as e: + logging.error(f"failed to build minio_host for nuvolaris: {e}") + return None + +def create_seaweedfs_nuv_storage(data): + """ + Creates nuvolaris SEAWEEDFS custom resources + """ + logging.info("*** configuring SEAWEEDFS storage for nuvolaris") + seaweedsfsClient = SeaweedfsClient() + + res = seaweedsfsClient.make_bucket("nuvolaris-data",data["default_bucket_quota"]) + if res: + openwhisk.annotate("s3_bucket_data=nuvolaris-data") + + res = seaweedsfsClient.add_anonymous_access() + res = seaweedsfsClient.make_bucket("nuvolaris-web",data["default_bucket_quota"]) + res = seaweedsfsClient.make_public_bucket("nuvolaris-web") + + if res: + openwhisk.annotate("s3_bucket_static=nuvolaris-web") + content_path = util.find_content_path("index.html") + + if(content_path): + logging.info(f"uploading example content to nuvolaris-web from {content_path}") + res = seaweedsfsClient.upload_folder_content(content_path,"nuvolaris-web") + else: + logging.warn("could not find example static content to upload") + + res = seaweedsfsClient.add_user(data["seaweedfs_nuv_user"],data["seaweedfs_nuv_user"],data["seaweedfs_nuv_password"],"nuvolaris-web,nuvolaris-data") + + if(res): + _annotate_nuv_metadata(data) + logging.info("*** configured SEAWEEDFS storage for nuvolaris") + +def create_ow_storage(state, ucfg: UserConfig, user_metadata: UserMetadata, owner=None): + seaweedfsClient = SeaweedfsClient() + namespace = ucfg.get("namespace") + secretkey = ucfg.get("object-storage.password") + + # assign default quota set for the user is not available + if not ucfg.exists('object-storage.quota'): + ucfg.put('object-storage.quota',cfg.get('seaweedfs.default-bucket-quota') or "1024") + logging.info(f"assigned bucket quota of {ucfg.get('object-storage.quota')}MB for namespace {namespace}") + + logging.info(f"*** configuring storage for namespace {namespace}") + buckets = [] + + if(ucfg.get('object-storage.data.enabled')): + bucket_name = ucfg.get('object-storage.data.bucket') + logging.info(f"*** adding private bucket {bucket_name} for {namespace}") + res = seaweedfsClient.make_bucket(bucket_name, ucfg.get('object-storage.quota')) + state['storage_data']=res + + if(res): + user_metadata.add_metadata("S3_BUCKET_DATA",bucket_name) + ucfg.put("S3_BUCKET_DATA",bucket_name) + buckets.append(bucket_name) + + if(ucfg.get('object-storage.route.enabled')): + bucket_name = ucfg.get("object-storage.route.bucket") + logging.info(f"*** adding public bucket {bucket_name} for {namespace}") + res = seaweedfsClient.make_bucket(bucket_name, ucfg.get('object-storage.quota')) + res = seaweedfsClient.make_public_bucket(bucket_name) + + if(res): + user_metadata.add_metadata("S3_BUCKET_STATIC",bucket_name) + ucfg.put("S3_BUCKET_STATIC",bucket_name) + buckets.append(bucket_name) + + content_path = util.find_content_path("index.html") + + if(content_path): + logging.info(f"uploading example content to {bucket_name} from {content_path}") + res = seaweedfsClient.upload_folder_content(content_path,bucket_name) + else: + logging.warn("could not find example static content to upload") + + state['storage_route']=res + + + res = seaweedfsClient.add_user(namespace,namespace, secretkey, ",".join(buckets)) + state['storage_user']=res + + if(res): + _add_seaweedfs_user_metadata(ucfg, user_metadata) + + return state + +def delete_ow_storage(ucfg): + seaweedfsClient = SeaweedfsClient() + namespace = ucfg.get("namespace") + + if(ucfg.get('object-storage.data.enabled')): + bucket_name = ucfg.get('object-storage.data.bucket') + logging.info(f"*** removing private bucket {bucket_name} for {namespace}") + seaweedfsClient.force_bucket_remove(bucket_name) + + if(ucfg.get('object-storage.route.enabled')): + bucket_name = ucfg.get("object-storage.route.bucket") + logging.info(f"*** removing public bucket {bucket_name} for {namespace}") + seaweedfsClient.force_bucket_remove(bucket_name) + + return seaweedfsClient.delete_user(namespace) + +def delete_by_owner(): + spec = kus.build("seaweedfs") + res = kube.delete(spec) + logging.info(f"delete seaweedfs: {res}") + return res + +def delete_by_spec(): + spec = cfg.get("state.seaweedfs.spec") + res = False + if spec: + res = kube.delete(spec) + logging.info(f"delete seaweedfs: {res}") + return res + +def delete(owner=None): + data = util.get_seaweedfs_config_data() + seaweedfs_ingress.delete_seaweedfs_ingresses(data, owner) + + if owner: + return delete_by_owner() + else: + return delete_by_spec() + +def patch(status, action, owner=None): + """ + Called by the operator patcher to create/delete seaweedfs component + """ + try: + logging.info(f"*** handling request to {action} seaweedfs") + if action == 'create': + msg = create(owner) + operator_util.patch_operator_status(status,'seaweedfs','on') + else: + msg = delete(owner) + operator_util.patch_operator_status(status,'seaweedfs','off') + + logging.info(msg) + logging.info(f"*** hanlded request to {action} seaweedfs") + except Exception as e: + logging.error('*** failed to update minio: %s' % e) + operator_util.patch_operator_status(status,'seaweedfs','error') + +def patch_ingresses(status, action, owner=None): + """ + Called by the operator patcher to create/delete seaweedfs component + """ + try: + logging.info(f"*** handling request to {action} seaweedfs ingresses") + data = util.get_minio_config_data() + if action == 'update': + msg = seaweedfs_ingress.create_seaweedfs_ingresses(data, owner) + operator_util.patch_operator_status(status,'seaweedfs-ingresses','on') + + logging.info(msg) + logging.info(f"*** hanlded request to {action} seaweedfs ingresses") + except Exception as e: + logging.error('*** failed to update minio seaweedfs: %s' % e) + operator_util.patch_operator_status(status,'seaweedfs-ingresses','error') diff --git a/nuvolaris/seaweedfs_ingress.py b/nuvolaris/seaweedfs_ingress.py new file mode 100644 index 0000000..1c0eb72 --- /dev/null +++ b/nuvolaris/seaweedfs_ingress.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import kopf, logging, time, os +import nuvolaris.kube as kube +import nuvolaris.kustomize as kus +import nuvolaris.config as cfg +import nuvolaris.util as util +import nuvolaris.apihost_util as apihost_util +import nuvolaris.endpoint as endpoint +import nuvolaris.openwhisk as openwhisk + +from nuvolaris.ingress_data import IngressData +from nuvolaris.route_data import RouteData + +def deploy_seaweedfs_route(apihost,namespace,type,service_name,port,context_path): + """ + Deploys a generic SEAWEEDFS route ingress + param: apihost + param: namespace + param: type (s3, console) + param: service_name (normally it is seaweedfs) + param: port (9090 or 9000) + paramL context_path (/) + """ + route = RouteData(apihost) + route.with_route_name(endpoint.api_route_name(namespace,type)) + route.with_service_name(service_name) + route.with_service_kind("Service") + route.with_service_port(port) + route.with_context_path(context_path) + + logging.info(f"*** configuring seaweedfs route for service {service_name}:{port}") + path_to_template_yaml = route.render_template(namespace) + res = kube.kubectl("apply", "-f",path_to_template_yaml) + os.remove(path_to_template_yaml) + return res + +def deploy_seaweedfs_ingress(apihost, namespace, type, service_name,port,context_path): + """ + Deploys a generic SEAWEEDFS nginx/traefik ingress + param: apihost + param: namespace + param: type (s3, console) + param: service_name (normally it is seaweedfs) + param: port (8888 or 8333) + paramL context_path (/) + """ + ingress = IngressData(apihost) + ingress.with_ingress_name(endpoint.api_ingress_name(namespace, type)) + ingress.with_secret_name(endpoint.ingress_secret_name(namespace, type)) + ingress.with_context_path(context_path) + ingress.with_service_name(service_name) + ingress.with_service_port(port) + + if ingress.requires_traefik_middleware(): + logging.info(f"*** configuring traefik middleware for {type} ingress") + path_to_template_yaml = ingress.render_traefik_middleware_template(namespace) + res = kube.kubectl("apply", "-f",path_to_template_yaml) + os.remove(path_to_template_yaml) + + logging.info(f"*** configuring static ingress for {type}") + path_to_template_yaml = ingress.render_template(namespace) + res = kube.kubectl("apply", "-f",path_to_template_yaml) + os.remove(path_to_template_yaml) + + return res + + +def create_s3_ingress_endpoint(data, runtime, apihost, owner=None): + """ + exposes SEAWEEDFS S3 api ingress ingress/route + """ + if runtime == 'openshift': + return deploy_seaweedfs_route(apihost,"nuvolaris","seaweedfs-s3","seaweedfs","9000","/") + else: + return deploy_seaweedfs_ingress(apihost,"nuvolaris","seaweedfs-s3","seaweedfs","9000","/") + +def create_console_ingress_endpoint(data, runtime, apihost, owner=None): + """ + exposes SEAWEEDFS api ingress ingress/route + """ + + if runtime == 'openshift': + return deploy_seaweedfs_route(apihost,"nuvolaris","seaweedfs-filer","seaweedfs","9090","/") + else: + return deploy_seaweedfs_ingress(apihost,"nuvolaris","seaweedfs-filer","seaweedfs","9090","/") + +def get_seaweedfs_ingress_hostname(runtime, apihost_url, prefix, hostname_from_config): + """ + Determine the SEAWEEDFS ingress hostname. In auto mode the prefix is appended + to the configured apihost,, otherwise the one from configuration is used. + """ + if hostname_from_config in ["auto"]: + return apihost_util.append_prefix_to_url(apihost_url, prefix) + + return apihost_util.get_ingress_url(runtime, hostname_from_config) + + +def create_seaweedfs_ingresses(data, owner=None): + """ + Creates all the SEAWEEDFS related ingresses according to provide configuration + """ + runtime = cfg.get('nuvolaris.kube') + apihost_url = apihost_util.get_apihost(runtime) + res = "" + + if data['seaweedfs_s3_ingress_enabled']: + s3_hostname_url = get_seaweedfs_ingress_hostname(runtime, apihost_url,"s3",data['seaweedfs_s3_ingress_hostname']) + res += create_s3_ingress_endpoint(data, runtime, s3_hostname_url, owner) + + if res: + openwhisk.annotate(f"s3_api_url={s3_hostname_url}") + + if data['seaweedfs_console_ingress_enabled']: + filer_hostname_url = get_seaweedfs_ingress_hostname(runtime, apihost_url,"filer",data['seaweedfs_console_ingress_hostname']) + res += create_console_ingress_endpoint(data, runtime, filer_hostname_url, owner) + + if res: + openwhisk.annotate(f"s3_console_url={filer_hostname_url}") + + return res + + +def delete_seaweedfs_ingress(runtime, namespace, ingress_class, type, owner=None): + """ + undeploys ingresses for seaweedfs apihost + """ + logging.info("*** removing ingresses for seaweedfs upload") + + try: + res = "" + if(runtime=='openshift'): + res = kube.kubectl("delete", "route",endpoint.api_route_name(namespace,type)) + return res + + res += kube.kubectl("delete", "ingress",endpoint.api_ingress_name(namespace,type)) + + if(ingress_class == 'traefik'): + res = kube.kubectl("delete", "middleware.traefik.containo.us",endpoint.api_middleware_ingress_name(namespace,type)) + + return res + except Exception as e: + logging.warn(e) + return None + +def delete_seaweedfs_ingresses(data, owner=None): + namespace = "nuvolaris" + runtime = cfg.get('nuvolaris.kube') + ingress_class = util.get_ingress_class(runtime) + res = "" + + if data['seaweedfs_s3_ingress_enabled']: + res += delete_seaweedfs_ingress(runtime, namespace, ingress_class, "seaweedfs-s3", owner) + + if data['seaweedfs_console_ingress_enabled']: + res += delete_seaweedfs_ingress(runtime, namespace, ingress_class, "seaweedfs-filer", owner) + + return res \ No newline at end of file diff --git a/nuvolaris/seaweedfs_util.py b/nuvolaris/seaweedfs_util.py new file mode 100644 index 0000000..89be2c1 --- /dev/null +++ b/nuvolaris/seaweedfs_util.py @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# this module wraps mc minio client using admin credentials +# to perform various operations + +import logging +import nuvolaris.config as cfg +import nuvolaris.template as ntp +import nuvolaris.util as util +import os +import nuvolaris.kube as kube +from types import NoneType +from typing import Optional + +from requests.exceptions import HTTPError + +import requests + +class SeaweedfsSimpleException(Exception): + def __init__(self, code: int, message: str): + self.message = message + self.code = code + + def __str__(self): + return f"{self.message} ({self.code})" + +class SeaweedfsUnauthorizedException(SeaweedfsSimpleException): + def __init__(self): + super().__init__(0, "Unauthorized") + +class SeaweedfsClient: + + def __init__(self): + self.filer_url = util.get_seaweedds_filer_host() + self.pod_name = util.get_pod_name_by_selector("app=seaweedfs","{.items[?(@.metadata.labels.app == 'seaweedfs')].metadata.name}") + + def _request(self, endpoint: str, json=None, method: str = "POST"): + headers = { + "Content-Type": "application/json", + } + url = f"{self.filer_url}/{endpoint}" + if json is None: + json = {} + response = requests.request(method, url, headers=headers, json=json) + try: + response.raise_for_status() + res = response.json() + if type(res) is not NoneType: + return res + else: + raise SeaweedfsSimpleException(code=res.get('code'), message=res.get('message')) + except HTTPError as e: + if e.response.status_code == 403: + raise SeaweedfsUnauthorizedException() + + def _multipart_request(self, endpoint: str, files=None, method: str = "POST"): + url = f"{self.filer_url}/{endpoint}" + if files is None: + files = {} + response = requests.request(method, url,files=files) + try: + response.raise_for_status() + res = response.json() + if type(res) is not NoneType: + return res + else: + raise SeaweedfsSimpleException(code=res.get('code'), message=res.get('message')) + except HTTPError as e: + if e.response.status_code == 403: + raise SeaweedfsUnauthorizedException() + + def _exec_weed_command(self,command): + logging.debug(f"executing command: {command} inside pod {self.pod_name}") + res = kube.kubectl("exec","-it",self.pod_name,"--","/bin/sh","-c",f"echo '{command}' | weed shell") + return res + + def make_bucket(self, bucket_name, quota_in_mb=None): + """ + adds a new bucket inside the configured seaweed instance + """ + res = util.check(self._exec_weed_command(f"s3.bucket.create -name {bucket_name}"),"make_bucket",True) + if quota_in_mb: + res = util.check(self._exec_weed_command(f"s3.bucket.quota -name {bucket_name} -op=set -sizeMB={quota_in_mb}"),"make_bucket",res) + return res + + def force_bucket_remove(self, bucket_name): + """ + removes unconditionally a bucket + """ + return util.check(self._exec_weed_command(f"s3.bucket.delete -name {bucket_name}"),"force_bucket_remove",True) + + def upload_folder_content(self,local_content,bucket): + """ + uploads the given content using a local alias for the corresponding bucket + """ + remote_file_name = os.path.basename(local_content) + url = f"{self.filer_url}/buckets/{bucket}/{remote_file_name}" + + with open(local_content, "rb") as f: + files = {"file": (remote_file_name, f)} + resp = requests.post(url, files=files) + + return util.check(resp.status_code==201,"upload_folder_content",True) + + def add_user(self, username, access_key, secret_key,buckets,actions="Read,Write,List,Tagging,Admin"): + """ + adds a new seaweedfs user using the filer api + """ + command = ( + f's3.configure -user={username} -access_key={access_key} -secret_key={secret_key} ' + f'-buckets={buckets} -actions={actions} -apply' + ) + return util.check(self._exec_weed_command(command),"add_user",True) + + def add_anonymous_access(self): + """ + adds a new seaweedfs user using the filer api + """ + command = 's3.configure -user=anonymous -actions=Read -apply' + return util.check(self._exec_weed_command(command),"add_user",True) + + def make_public_bucket(self, bucket_name): + """ + assign the specified buckets to the given users + """ + return util.check(self._multipart_request(f"buckets/{bucket_name}?public=1", method="PUT"),"make_public_bucket",True) + + def make_private_bucket(self, bucket_name): + """ + assign the specified buckets to the given users + """ + return util.check(self._multipart_request(f"buckets/{bucket_name}/?public=0", method="PUT"),"make_private_bucket",True) + + def delete_user(self, username): + """ + removes a user from seaweedfs + """ + return util.check(self._exec_weed_command(f"s3.user.delete -name {username}"),"delete_user",True) diff --git a/nuvolaris/templates/milvus-cfg-base.yaml b/nuvolaris/templates/milvus-cfg-base.yaml index 2114672..2190c80 100644 --- a/nuvolaris/templates/milvus-cfg-base.yaml +++ b/nuvolaris/templates/milvus-cfg-base.yaml @@ -50,8 +50,8 @@ data: type: etcd minio: - address: nuvolaris-minio - port: 9000 + address: {{bucket_server_hostname}} + port: {{bucket_server_port}} accessKeyID: {{milvus_s3_username}} secretAccessKey: {{milvus_s3_password}} useSSL: false diff --git a/nuvolaris/templates/milvus-cfg-slim-base.yaml b/nuvolaris/templates/milvus-cfg-slim-base.yaml index 27d9758..103a191 100644 --- a/nuvolaris/templates/milvus-cfg-slim-base.yaml +++ b/nuvolaris/templates/milvus-cfg-slim-base.yaml @@ -49,8 +49,8 @@ data: type: etcd minio: - address: nuvolaris-minio - port: 9000 + address: {{bucket_server_hostname}} + port: {{bucket_server_port}} accessKeyID: {{milvus_s3_username}} secretAccessKey: {{milvus_s3_password}} useSSL: false diff --git a/nuvolaris/templates/redis-set.yaml b/nuvolaris/templates/redis-set.yaml index 836b57a..731b4be 100644 --- a/nuvolaris/templates/redis-set.yaml +++ b/nuvolaris/templates/redis-set.yaml @@ -42,7 +42,7 @@ spec: {% endif %} containers: - name: redis - image: bitnami/valkey:7.2.5 + image: bitnamisecure/valkey:latest command: ["/bin/sh","-c","redis-server /redis-master/redis.conf"] env: - name: MASTER diff --git a/nuvolaris/user_config.py b/nuvolaris/user_config.py index c8393ad..d1a37bd 100644 --- a/nuvolaris/user_config.py +++ b/nuvolaris/user_config.py @@ -17,6 +17,7 @@ # import flatdict, json, os import logging +import nuvolaris.config as cfg class UserConfig: _config = {} diff --git a/nuvolaris/user_handlers.py b/nuvolaris/user_handlers.py index 5ab1f6a..0ffaf1f 100644 --- a/nuvolaris/user_handlers.py +++ b/nuvolaris/user_handlers.py @@ -33,6 +33,7 @@ import nuvolaris.storage_static as static import nuvolaris.user_patcher as user_patcher import nuvolaris.userdb_util as userdb +import nuvolaris.seaweedfs_deploy as seaweedfs from nuvolaris.quota_checker import REDIS_DB_QUOTA_ANNOTATION from nuvolaris.user_config import UserConfig from nuvolaris.user_metadata import UserMetadata @@ -71,7 +72,10 @@ def whisk_user_create(spec, name, patch, **kwargs): if(cfg.get('components.minio') and (ucfg.get('object-storage.data.enabled') or ucfg.get('object-storage.route.enabled'))): minio_deploy.create_ow_storage(state, ucfg, user_metadata, owner) - if(cfg.get('components.minio') and ucfg.get('object-storage.route.enabled') and cfg.get('components.static')): + if(cfg.get('components.seaweedfs') and (ucfg.get('object-storage.data.enabled') or ucfg.get('object-storage.route.enabled'))): + seaweedfs.create_ow_storage(state, ucfg, user_metadata, owner) + + if((cfg.get('components.minio') or cfg.get('components.seaweedfs')) and ucfg.get('object-storage.route.enabled') and cfg.get('components.static')): res = static.create_ow_static_endpoint(ucfg,user_metadata, owner) logging.info("OpenWhisk static endpoint for %s added = %s", ucfg.get('namespace'), res) state['static']= res @@ -126,7 +130,11 @@ def whisk_user_delete(spec, name, **kwargs): res = minio_deploy.delete_ow_storage(ucfg) logging.info(f"OpenWhisk namespace {ucfg.get('namespace')} MINIO storage removed = {res}") - if(cfg.get('components.minio') and ucfg.get('object-storage.route.enabled') and cfg.get('components.static')): + if(cfg.get('components.seaweedfs') and (ucfg.get('object-storage.data.enabled') or ucfg.get('object-storage.route.enabled'))): + res = seaweedfs.delete_ow_storage(ucfg) + logging.info(f"OpenWhisk namespace {ucfg.get('namespace')} SEAWEEDFS storage removed = {res}") + + if((cfg.get('components.minio') or cfg.get('components.seaweedfs')) and ucfg.get('object-storage.route.enabled') and cfg.get('components.static')): res = static.delete_ow_static_endpoint(ucfg) logging.info(f"OpenWhisk static endpoint for {ucfg.get('namespace')} removed = {res}") diff --git a/nuvolaris/util.py b/nuvolaris/util.py index f2b6eb6..009c0c4 100644 --- a/nuvolaris/util.py +++ b/nuvolaris/util.py @@ -21,6 +21,7 @@ import random import time import uuid +import os from base64 import b64decode, b64encode from typing import List, Union from urllib.parse import urlparse @@ -523,8 +524,10 @@ def get_storage_static_config_data(): minio_port=cfg.get('minio.port') or "9000" data['storage_url']=f"http://{minio_host}.nuvolaris.svc.cluster.local:{minio_port}" - if cfg.get('components.cosi'): - data['storage_url']=apihost_util.add_suffix_to_url(get_object_storage_rgw_url(),"cluster.local") + if cfg.get('components.seaweedfs'): + seaweedfs_api_host = cfg.get("seaweedfs.host") or "seaweedfs" + seaweedfs_api_port = cfg.get("seaweedfs.port") or "9000" + data['storage_url']=f"http://{seaweedfs_api_host}.nuvolaris.svc.cluster.local:{seaweedfs_api_port}" storage_static_affinity_tolerations_data(data) return data @@ -588,7 +591,12 @@ def postgres_backup_affinity_tolerations_data(data): # populate specific affinity data for registry def registry_affinity_tolerations_data(data): common_affinity_tolerations_data(data) - data["pod_anti_affinity_name"] = "registry" + data["pod_anti_affinity_name"] = "registry" + +# populate specific affinity data for seaweedfs +def seaweedfs_affinity_tolerations_data(data): + common_affinity_tolerations_data(data) + data["pod_anti_affinity_name"] = "seaweedfs" # wait for a pod name using a label selector and eventually an optional jsonpath @nuv_retry() @@ -797,6 +805,7 @@ def get_milvus_config_data(): 'milvus_s3_username': 'miniomilvus', 'milvus_s3_password': cfg.get('milvus.password.s3') or "s0meP@ass3", 'milvus_bucket_name': 'vectors', + 'milvus_bucket_quota': cfg.get('milvus.volume-size.bucket') or 10240, 'milvus_bucket_prefix': 'milvus/nuvolaris-milvus', 'size': cfg.get('milvus.volume-size.cluster') or 10, 'zookeeper_size': cfg.get('milvus.volume-size.zookeeper') or 10, @@ -813,6 +822,14 @@ def get_milvus_config_data(): 'milvus_max_database_num': cfg.get('milvus.root-coord.max-database-num') or 64, 'slim': cfg.get('nuvolaris.slim') or False, } + + if cfg.get('components.minio'): + data["bucket_server_hostname"]="nuvolaris-minio" + data["bucket_server_port"]="9000" + + if cfg.get('components.seaweedfs'): + data["bucket_server_hostname"]="seaweedfs" + data["bucket_server_port"]="9000" data["etcd_range"]=range(data["etcd_replicas"]) milvus_standalone_affinity_tolerations_data(data) @@ -842,4 +859,35 @@ def get_registry_config_data(): registry_affinity_tolerations_data(data) return data +def find_content_path(filename): + absolute_path = os.path.dirname(__file__) + relative_path = "../deploy/content" + return os.path.join(absolute_path, relative_path, filename) + +# return seaweedfs configuration parameters with default values if not configured +def get_seaweedfs_config_data(): + data = { + "applypodsecurity":get_enable_pod_security(), + "name":"seaweedfs", + "container":"seaweedfs", + "seaweedfs_host": cfg.get('seaweedfs.host') or 'seaweedfs', + "size": cfg.get('seaweedfs.volume-size') or "60", + "default_bucket_quota": cfg.get('seaweedfs.default-bucket-quota') or "1024", + "storage_class": cfg.get("nuvolaris.storageclass"), + "pvcName":"seaweedfs-pvc", + "seaweedfs_nuv_user": cfg.get('seaweedfs.nuvolaris.user') or "nuvolaris", + "seaweedfs_nuv_password": cfg.get('seaweedfs.nuvolaris.password') or "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "seaweedfs_s3_ingress_enabled": cfg.get('seaweedfs.ingress.s3-enabled') or False, + "seaweedfs_console_ingress_enabled": cfg.get('seaweedfs.ingress.console-enabled') or False, + "seaweedfs_s3_ingress_hostname": cfg.get('seaweedfs.ingress.s3-hostname') or "auto", + "seaweedfs_console_ingress_hostname": cfg.get('seaweedfs.ingress.console-hostname') or "auto" + } + seaweedfs_affinity_tolerations_data(data) + return data + +def get_seaweedds_filer_host(): + seaweedfs_filer_host = cfg.get("seaweedfs.host", "SEAWEEDFS_API_HOST", "seaweedfs") + seaweedfs_filer_port = cfg.get("seaweedfs.port", "SEAWEEDFS_API_PORT", "9090") + return f"http://{seaweedfs_filer_host}:{seaweedfs_filer_port}" + diff --git a/tests/kind/seaweedfs_test.ipy b/tests/kind/seaweedfs_test.ipy new file mode 100644 index 0000000..d7b7e0e --- /dev/null +++ b/tests/kind/seaweedfs_test.ipy @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + +import nuvolaris.config as cfg +import nuvolaris.kube as kube +import nuvolaris.seaweedfs_deploy as seaweedfs_deploy +import nuvolaris.testutil as tu +import nuvolaris.util as util +import logging + +logging.basicConfig(level=logging.DEBUG) + +tu.run_proc("kubectl -n nuvolaris delete all --all") +tu.run_proc("kubectl -n nuvolaris delete pvc --all") + +# for this test seaweedfsClient should see this env variable +os.environ['SEAWEEDFS_API_HOST']='localhost' +os.environ['SEAWEEDFS_API_PORT']='9090' + +# test +assert(cfg.configure(tu.load_sample_config())) +assert(cfg.detect_storage()["nuvolaris.storageclass"]) + +# for this test minioClient should see this env variable +assert(seaweedfs_deploy.create()) + +pod_name = util.get_pod_name("{.items[?(@.metadata.labels.app == 'seaweedfs')].metadata.name}") +assert(pod_name) + +assert(seaweedfs_deploy.delete()) \ No newline at end of file diff --git a/tests/kind/whisk-minimal.yaml b/tests/kind/whisk-minimal.yaml index 8051cd5..c3fc351 100644 --- a/tests/kind/whisk-minimal.yaml +++ b/tests/kind/whisk-minimal.yaml @@ -33,17 +33,25 @@ spec: # start mongodb mongodb: false # start redis - redis: false + redis: true # start cron based action parser cron: false # tls enabled or not tls: false # minio enabled or not - minio: true + minio: false # minio static enabled or not static: true # postgres enabled or not - postgres: false + postgres: false + # etcd enabled or not + etcd: false + # milvus enabled or not + milvus: false + # registry enabled or not + registry: false + # seaweedfs enabled or not + seaweedfs: true openwhisk: namespaces: whisk-system: 789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP @@ -54,6 +62,7 @@ spec: provisioner: auto #rancher.io/local-path ingressclass: auto #nginx ingresslb: auto #ingress-nginx/ingress-nginx-controller + slim: true couchdb: host: couchdb port: 5984 @@ -123,11 +132,33 @@ spec: nuvolaris: user: nuvolaris password: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG - postgres: - volume-size: 5 - replicas: 2 - admin: - password: 0therPa55 - replica-password: 0therPa55RR + milvus: + volume-size: + cluster: 20 + zookeeper: 15 + journal: 20 + ledgers: 25 + replicas: 1 + password: + root: An0therPa55 + etcd: 97Vk2{qe8o>S + s3: 8_d$8zCrl7£m + registry: + mode: internal + volume-size: 20 + auth: + username: nuvolaris + password: 4pwdregistry + hostname: auto + ingress: + enabled: false + seaweedfs: + volume-size: 60 nuvolaris: - password: s0meP@ass3 + user: nuvolaris + password: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + ingress: + s3-enabled: true + s3-hostname: auto + console-enabled: false + console-hostname: auto diff --git a/tests/kind/whisk-slim.yaml b/tests/kind/whisk-slim.yaml index 5c41565..f6f3f4a 100644 --- a/tests/kind/whisk-slim.yaml +++ b/tests/kind/whisk-slim.yaml @@ -51,7 +51,9 @@ spec: # milvus enabled or not milvus: true # registry enabled or not - registry: true + registry: false + # seaweedfs enabled or not + seaweedfs: false openwhisk: namespaces: whisk-system: 789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP @@ -167,4 +169,14 @@ spec: password: 4pwdregistry hostname: auto ingress: - enabled: false \ No newline at end of file + enabled: false + seaweedfs: + volume-size: 60 + nuvolaris: + user: nuvolaris + password: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + ingress: + s3-enabled: true + s3-hostname: auto + console-enabled: false + console-hostname: auto \ No newline at end of file diff --git a/tests/whisk.yaml b/tests/whisk.yaml index 5f874b0..0570958 100644 --- a/tests/whisk.yaml +++ b/tests/whisk.yaml @@ -51,7 +51,9 @@ spec: # milvus enabled or not milvus: true # registry enabled or not - registry: true + registry: true + # seaweedfs enabled or not + seaweedfs: false openwhisk: namespaces: whisk-system: 789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP @@ -156,4 +158,15 @@ spec: password: 4pwdregistry hostname: auto ingress: - enabled: false + enabled: false + seaweedfs: + volume-size: 60 + nuvolaris: + user: nuvolaris + password: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + ingress: + s3-enabled: true + s3-hostname: auto + console-enabled: false + console-hostname: auto +