diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml index 9dbf9e742b3..260405a1165 100644 --- a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml +++ b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml @@ -3,11 +3,7 @@ kind: Kustomization namespace: kubeflow resources: -- seaweedfs-deployment.yaml -- seaweedfs-pvc.yaml -- seaweedfs-networkpolicy.yaml - seaweedfs-create-admin-user-job.yaml -- seaweedfs-service.yaml - seaweedfs-service-account.yaml - minio-service.yaml - mlpipeline-minio-artifact-secret.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-create-admin-user-job.yaml b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-create-admin-user-job.yaml index 0fcbfe00e2e..51f245a066b 100644 --- a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-create-admin-user-job.yaml +++ b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-create-admin-user-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: init-seaweedfs - image: 'chrislusf/seaweedfs:3.85' + image: 'chrislusf/seaweedfs:3.92' env: - name: WEED_CLUSTER_DEFAULT value: "sw" @@ -44,7 +44,7 @@ spec: echo "Service at $url failed to become ready within 5 minutes" exit 1 } - wait_for_service "http://minio-service.kubeflow:9000/status" + wait_for_service "http://minio-service:9000/status" echo "Creating S3 bucket..." echo "s3.bucket.create --name mlpipeline" | /usr/bin/weed shell > /dev/null 2>&1 if [ $? -eq 0 ]; then @@ -64,14 +64,17 @@ spec: echo "Failed to configure S3 credentials" exit 1 fi - securityContext: # Using restricted profile - allowPrivilegeEscalation: false - privileged: false - runAsNonRoot: true + ports: + - containerPort: 9333 + name: http-master + - containerPort: 19333 + #securityContext: # Using restricted profile + #allowPrivilegeEscalation: false + #privileged: false + #runAsNonRoot: true # image defaults to root user - runAsUser: 1001 - runAsGroup: 1001 - capabilities: - drop: - - ALL + #runAsUser: 1001 + #runAsGroup: 1001 + ## drop: + #- ALL serviceAccountName: seaweedfs diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml new file mode 100644 index 00000000000..761d39d63a6 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer +spec: + serviceName: seaweedfs-filer + replicas: 2 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: filer + template: + metadata: + labels: + app: seaweedfs + component: filer + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: seaweedfs + component: filer + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1001 + containers: + - name: seaweedfs-filer + image: 'chrislusf/seaweedfs:3.92' + args: + - 'filer' + - '-port=8888' + - '-iam' + - '-master=seaweedfs-master-0.seaweedfs-master:9333,seaweedfs-master-1.seaweedfs-master:9333,seaweedfs-master-2.seaweedfs-master:9333' + volumeMounts: + - name: data-filer + mountPath: /data + ports: + - containerPort: 8888 + name: http-filer + - containerPort: 18888 + name: grpc-filer + - containerPort: 8333 + name: http-s3 + - containerPort: 8111 + name: http-iam + readinessProbe: + httpGet: + path: / + port: 8888 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 2Gi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1001 + runAsGroup: 1001 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - metadata: + name: data-filer + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml new file mode 100644 index 00000000000..e4fb1e11d90 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer-intern +spec: + ports: + - name: grpc-filer + port: 18888 + protocol: TCP + targetPort: 18888 + - name: http-filer + port: 8888 + protocol: TCP + targetPort: 8888 + publishNotReadyAddresses: true + selector: + app: seaweedfs + component: filer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer + namespace: kubeflow +spec: + ports: + - name: http-iam + port: 8111 + protocol: TCP + targetPort: 8111 + - name: http-s3 + port: 8333 + protocol: TCP + targetPort: 8333 + - name: grpc-filer + port: 18888 + protocol: TCP + targetPort: 18888 + - name: http-filer + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + app: seaweedfs + component: filer \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml new file mode 100644 index 00000000000..8197c845e17 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- ../../base +- filer-statefulset.yaml +- filer-svc.yaml +- master-statefulset.yaml +- master-svc.yaml +- s3-gateway-deployment.yaml +- volume-statefulset.yaml +- volume-svc.yaml + +patches: +- target: + version: v1 + kind: Job + name: init-seaweedfs + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/1/value + value: "seaweedfs-master:9333" + - op: add + path: /spec/template/spec/containers/0/env/- + value: {"name": "WEED_CLUSTER_SW_FILER", "value": "seaweedfs-filer:8888"} +- target: + version: v1 + kind: Service + name: minio-service + patch: |- + - op: add + path: /spec/selector/component + value: s3 diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml new file mode 100644 index 00000000000..a07c5afc7a2 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: master + name: seaweedfs-master +spec: + serviceName: seaweedfs-master + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: master + template: + metadata: + labels: + app: seaweedfs + component: master + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: seaweedfs + component: master + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1001 + containers: + - name: seaweedfs-master + image: 'chrislusf/seaweedfs:3.92' + args: + - 'master' + - '-mdir=/data' + - '-defaultReplication=001' + - '-volumePreallocate=false' + - '-ip=$(POD_NAME).seaweedfs-master.$(NAMESPACE)' + - '-ip.bind=0.0.0.0' + - '-port=9333' + - '-peers=seaweedfs-master-0.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-1.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-2.seaweedfs-master.$(NAMESPACE):9333' + volumeMounts: + - name : data-master + mountPath: /data + ports: + - containerPort: 9333 + name: http-master + - containerPort: 19333 + name: grpc-master + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + readinessProbe: + httpGet: + path: /cluster/status + port: 9333 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + resources: + requests: + cpu: 128m + memory: 256Mi + limits: + memory: 256Mi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1001 + runAsGroup: 1001 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - metadata: + name: data-master + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml new file mode 100644 index 00000000000..d108363f633 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: master + name: seaweedfs-master +spec: + publishNotReadyAddresses: true + ports: + - name: http-master + port: 9333 + protocol: TCP + targetPort: 9333 + - name: grpc-master + port: 19333 + protocol: TCP + targetPort: 19333 + selector: + app: seaweedfs + component: master \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml new file mode 100644 index 00000000000..3d8ae82bee7 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seaweedfs-s3 + namespace: kubeflow + labels: + app: seaweedfs + component: s3 +spec: + replicas: 2 + selector: + matchLabels: + app: seaweedfs + component: s3 + template: + metadata: + labels: + app: seaweedfs + component: s3 + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: seaweedfs + containers: + - name: seaweedfs + image: chrislusf/seaweedfs:3.92 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - 's3' + - '-ip.bind=0.0.0.0' + - '-filer=seaweedfs-filer:8888' + ports: + - containerPort: 8333 + name: swfs-s3 + readinessProbe: + httpGet: + path: /status + port: 8333 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 50 + timeoutSeconds: 2 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1001 + runAsGroup: 1001 + capabilities: + drop: + - ALL \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml new file mode 100644 index 00000000000..4c2082e42f2 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: volume + name: seaweedfs-volume +spec: + serviceName: seaweedfs-volume + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: volume + template: + metadata: + labels: + app: seaweedfs + component: volume + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: seaweedfs + component: volume + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 150 + securityContext: + fsGroup: 1001 + containers: + - name: seaweedfs + image: chrislusf/seaweedfs:3.92 + args: + - volume + - -port=8080 + - -dir /data1 + - -ip.bind=0.0.0.0 + - -ip=$(POD_NAME).seaweedfs-volume.$(NAMESPACE) + - -mserver=seaweedfs-master-0.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-1.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-2.seaweedfs-master.$(NAMESPACE):9333 + volumeMounts: + - mountPath: /data1/ + name: swfs-vol1 + ports: + - containerPort: 8080 + name: swfs-vol + protocol: TCP + - containerPort: 18080 + name: swfs-vol-grpc + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: WEED_CLUSTER_SW_FILER + value: seaweedfs-filer:8888 + - name: WEED_CLUSTER_SW_MASTER + value: seaweedfs-master:9333 + readinessProbe: + failureThreshold: 100 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 30 + resources: + requests: + cpu: 128m + memory: 256Mi + limits: + memory: 2Gi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1001 + runAsGroup: 1001 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: swfs-vol1 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + volumeMode: Filesystem \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml new file mode 100644 index 00000000000..50f4f715e54 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: volume + name: seaweedfs-volume +spec: + ports: + - name: swfs-volume + port: 8080 + protocol: TCP + targetPort: 8080 + - name: swfs-volume-18080 + port: 18080 + protocol: TCP + targetPort: 18080 + selector: + app: seaweedfs + component: volume \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml b/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml new file mode 100644 index 00000000000..bdc54af3246 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml @@ -0,0 +1,33 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: seaweedfs-service +spec: + action: ALLOW + selector: + matchLabels: + app: seaweedfs + component: s3 + rules: + - from: + - source: + principals: + - cluster.local/ns/kubeflow/sa/ml-pipeline + - from: + - source: + principals: + - cluster.local/ns/kubeflow/sa/ml-pipeline-ui + # Allow traffic to s3 endpoint from User Pipeline Pods, which don't have a sidecar. + # Also needed for traffic from seaweedfs init pod. Seaweedfs gives the client an ip to connect to. This can not be + # handled well by istio (AuthPolicy). Instead, access to the sensitive ports will be limited by the NetworkPolicy. + - {} +--- +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: ml-pipeline-seaweedfs +spec: + host: seaweedfs.kubeflow.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL diff --git a/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml new file mode 100644 index 00000000000..449e3cdd912 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- ../base +patches: +- target: + version: security.istio.io/v1beta1 + kind: AuthorizationPolicy + name: seaweedfs-service + patch: |- + - op: add + path: /spec/selector/matchLabels/component + value: s3 diff --git a/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml new file mode 100644 index 00000000000..719e1505bf3 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base/ +- seaweedfs-deployment.yaml +- seaweedfs-networkpolicy.yaml +- seaweedfs-pvc.yaml +- seaweedfs-service.yaml \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-deployment.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-deployment.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-deployment.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-deployment.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-networkpolicy.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-networkpolicy.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-networkpolicy.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-networkpolicy.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-pvc.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-pvc.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-pvc.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-pvc.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-service.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-service.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-service.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-service.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/istio/istio-authorization-policy.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/istio-authorization-policy.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/istio/istio-authorization-policy.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/istio/istio-authorization-policy.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml similarity index 92% rename from manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml index 2dffd5d150b..f948da42bc4 100644 --- a/manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml +++ b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml @@ -3,5 +3,5 @@ kind: Kustomization namespace: kubeflow resources: -- ../base/ +- ../base - istio-authorization-policy.yaml