feat: fix webhook bug, create tfconn in pod controller (#7)

Author: 0x5457

.github/workflows/release.yml

@@ -11,6 +11,8 @@ jobs:
     permissions:
       #  to create release tags (cycjimmy/semantic-release-action)
       contents: write
+      issues: write
+      pull-requests: write
 
     runs-on: ubuntu-latest
     outputs:
@@ -43,6 +45,7 @@ jobs:
         with:
           images: tensorfusion/tensor-fusion-operator
           tags: type=semver,pattern={{version}},value=${{needs.release.outputs.version}}
+
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:

.mirrord/mirrord.json

@@ -0,0 +1,17 @@
+{
+    "feature": {
+        "network": {
+            "incoming": "steal",
+            "outgoing": true
+        },
+        "fs": "read",
+        "env": true
+    },
+    "target": {
+        "namespace": "tensor-fusion",
+        "path": {
+            "deployment": "tensor-fusion-operator-controller-manager",
+            "container": "manager"
+        }
+    }
+}

Makefile

@@ -14,7 +14,7 @@ endif
 # Be aware that the target commands are only tested with Docker which is
 # scaffolded by default. However, you might want to replace it to use other
 # tools. (i.e. podman)
-CONTAINER_TOOL ?= docker
+CONTAINER_TOOL ?= $(shell command -v docker >/dev/null 2>&1 && echo docker || echo nerdctl)
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.

PROJECT

@@ -23,7 +23,8 @@ resources:
   kind: GPU
   path: github.com/NexusGPU/tensor-fusion-operator/api/v1
   version: v1
-- core: true
+- controller: true
+  core: true
   group: core
   kind: Pod
   path: k8s.io/api/core/v1

cmd/main.go

@@ -67,6 +67,8 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
+	var configFile string
+	flag.StringVar(&configFile, "config", "/etc/tensor-fusion/config.yaml", "Config file of tensor-fusion-operator")
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -152,7 +154,13 @@ func main() {
 	}
 
 	ctx := context.Background()
-	config := config.NewDefaultConfig()
+	config, err := config.LoadConfig(configFile)
+	if os.IsNotExist(err) {
+		setupLog.Info("config file is not exists, use default config", "configFile", configFile)
+	} else if err != nil {
+		setupLog.Error(err, "unable to load config", "configFile", configFile, "err", err)
+		os.Exit(1)
+	}
 	scheduler := scheduler.NewNaiveScheduler()
 	if err = (&controller.TensorFusionConnectionReconciler{
 		Client:    mgr.GetClient(),
@@ -183,6 +191,7 @@ func main() {
 		}
 	}
 
+
 	if err = (&controller.TensorFusionClusterReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
@@ -211,6 +220,13 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "GPUNodeClass")
 		os.Exit(1)
 	}
+  if err = (&controller.PodReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Pod")
+    os.Exit(1)
+  }
 	// +kubebuilder:scaffold:builder
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")

config/default/kustomization.yaml

@@ -1,5 +1,5 @@
 # Adds namespace to all resources.
-namespace: tensor-fusion-operator-system
+namespace: tensor-fusion
 
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named

config/default/manager_webhook_patch.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: controller-manager
-  namespace: system
   labels:
     app.kubernetes.io/name: tensor-fusion-operator
     app.kubernetes.io/managed-by: kustomize

config/manager/kustomization.yaml

@@ -2,3 +2,9 @@ namespace: tensor-fusion
 
 resources:
 - manager.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: tensorfusion/tensor-fusion-operator
+  newTag: latest

config/manager/manager.yaml

@@ -1,17 +1,7 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    control-plane: controller-manager
-    app.kubernetes.io/name: tensor-fusion-operator
-    app.kubernetes.io/managed-by: kustomize
-  name: system
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: controller-manager
-  namespace: system
   labels:
     control-plane: controller-manager
     app.kubernetes.io/name: tensor-fusion-operator
@@ -28,35 +18,6 @@ spec:
       labels:
         control-plane: controller-manager
     spec:
-      # TODO(user): Uncomment the following code to configure the nodeAffinity expression
-      # according to the platforms which are supported by your solution.
-      # It is considered best practice to support multiple architectures. You can
-      # build your manager image using the makefile target docker-buildx.
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #           - key: kubernetes.io/arch
-      #             operator: In
-      #             values:
-      #               - amd64
-      #               - arm64
-      #               - ppc64le
-      #               - s390x
-      #           - key: kubernetes.io/os
-      #             operator: In
-      #             values:
-      #               - linux
-      securityContext:
-        runAsNonRoot: true
-        # TODO(user): For common cases that do not require escalating privileges
-        # it is recommended to ensure that all your Pods/Containers are restrictive.
-        # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
-        # Please uncomment the following code if your project does NOT have to work on old Kubernetes
-        # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ).
-        # seccompProfile:
-        #   type: RuntimeDefault
       containers:
       - command:
         - /manager
@@ -65,11 +26,6 @@ spec:
           - --health-probe-bind-address=:8081
         image: controller:latest
         name: manager
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - "ALL"
         livenessProbe:
           httpGet:
             path: /healthz
@@ -82,8 +38,6 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
-        # TODO(user): Configure the resources accordingly based on the project requirements.
-        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
             cpu: 500m

config/rbac/role.yaml

@@ -4,6 +4,18 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - tensor-fusion.ai
   resources: