Skip to content

Commit 63d09a6

Browse files
fressivefressive
committed
update: Container service
1 parent 3bdfd60 commit 63d09a6

File tree

7 files changed

+199
-54
lines changed

7 files changed

+199
-54
lines changed

‎internal/k8s/container_manager.go‎

Lines changed: 97 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -36,49 +36,58 @@ type ContainerManager struct {
3636
Store *store.Store
3737
}
3838

39-
func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *store.User, flag string, store *store.Store) (string, error) {
40-
hash := util.SHA256(c.UUID + u.UUID)[:16]
39+
type ContainerCreatePayload struct {
40+
Image string `json:"image" validate:"required"`
41+
MemoryLimit string `json:"memory_limit" default:"128Mi"`
42+
CPULimit string `json:"cpu_limit" default:"100m"`
43+
StorageLimit string `json:"storage_limit" default:"1Gi"`
44+
ExposedPort int `json:"exposed_port"`
45+
RegistryAccessTokenUUID string `json:"registry_access_token_uuid"`
46+
}
47+
48+
type ContainerInfo struct {
49+
Identifier string `json:"-"`
50+
Labels map[string]string `json:"-"`
51+
Flag string `json:"-"`
52+
}
53+
54+
func (cm *ContainerManager) CreateContainer(payload *ContainerCreatePayload, info *ContainerInfo, store *store.Store) (string, string, error) {
55+
hash := util.SHA256(info.Identifier)[:16]
4156
name := "container-" + hash
4257
// Create a container in k8s
58+
59+
info.Labels["identifier"] = info.Identifier
60+
4361
deployment := &appv1.Deployment{
4462
ObjectMeta: metav1.ObjectMeta{
45-
Name: name,
46-
Labels: map[string]string{
47-
"challenge": c.UUID,
48-
"user": u.UUID,
49-
},
63+
Name: name,
64+
Labels: info.Labels,
5065
},
5166
Spec: appv1.DeploymentSpec{
5267
Replicas: ptr.To[int32](1),
5368
Selector: &metav1.LabelSelector{
54-
MatchLabels: map[string]string{
55-
"challenge": c.UUID,
56-
"user": u.UUID,
57-
},
69+
MatchLabels: info.Labels,
5870
},
5971
Template: v1.PodTemplateSpec{
6072
ObjectMeta: metav1.ObjectMeta{
61-
Labels: map[string]string{
62-
"challenge": c.UUID,
63-
"user": u.UUID,
64-
},
73+
Labels: info.Labels,
6574
},
6675
Spec: v1.PodSpec{
6776
Containers: []v1.Container{
6877
{
6978
Name: "challenge",
70-
Image: c.Image.Name,
79+
Image: payload.Image,
7180
Env: []v1.EnvVar{
7281
{
7382
Name: "HOSHINO_FLAG",
74-
Value: flag,
83+
Value: info.Flag,
7584
},
7685
},
7786
Resources: v1.ResourceRequirements{
7887
Limits: v1.ResourceList{
79-
v1.ResourceCPU: resource.MustParse(c.Image.CPULimit),
80-
v1.ResourceMemory: resource.MustParse(c.Image.MemoryLimit),
81-
v1.ResourceLimitsEphemeralStorage: resource.MustParse(c.Image.StorageLimit),
88+
v1.ResourceCPU: resource.MustParse(payload.CPULimit),
89+
v1.ResourceMemory: resource.MustParse(payload.MemoryLimit),
90+
v1.ResourceEphemeralStorage: resource.MustParse(payload.StorageLimit),
8291
},
8392
},
8493
},
@@ -93,28 +102,22 @@ func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *stor
93102
if err != nil {
94103
// handle error
95104
slog.Error(err.Error())
96-
return "", err
105+
return "", "", err
97106
}
98107

99108
slog.Info(fmt.Sprintf("Deployment created: %s", deploymentObj.Name))
100109

101110
service := &v1.Service{
102111
ObjectMeta: metav1.ObjectMeta{
103-
Name: name + "-service",
104-
Labels: map[string]string{
105-
"challenge": c.UUID,
106-
"user": u.UUID,
107-
},
112+
Name: name + "-service",
113+
Labels: info.Labels,
108114
},
109115
Spec: v1.ServiceSpec{
110-
Selector: map[string]string{
111-
"challenge": c.UUID,
112-
"user": u.UUID,
113-
},
116+
Selector: info.Labels,
114117
Ports: []v1.ServicePort{
115118
{
116-
Port: int32(c.ExposedPort),
117-
TargetPort: intstr.FromInt(c.ExposedPort),
119+
Port: int32(payload.ExposedPort),
120+
TargetPort: intstr.FromInt(payload.ExposedPort),
118121
},
119122
},
120123
},
@@ -124,25 +127,49 @@ func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *stor
124127
if err != nil {
125128
// handle error
126129
slog.Error(err.Error())
127-
return "", err
130+
return "", "", err
128131
}
129132

130133
slog.Info(fmt.Sprintf("Service created: %s, %d", serviceObj.Name, serviceObj.Spec.Ports[0].NodePort))
131134

132135
containerUUID := util.UUID()
133136

137+
pods, err := cm.K8SClient.CoreV1().Pods("challenge-containers").List(context.Background(), metav1.ListOptions{
138+
LabelSelector: fmt.Sprintf("identifier=%s", info.Identifier),
139+
})
140+
141+
if err != nil {
142+
// handle error
143+
slog.Error(err.Error())
144+
return "", "", err
145+
}
146+
147+
if len(pods.Items) == 0 {
148+
slog.Warn("No pod found")
149+
return "", "", fmt.Errorf("No pod found")
150+
}
151+
152+
pod := pods.Items[0]
153+
nodeName := pod.Spec.NodeName
154+
nodeDomain := ""
155+
node, err := cm.K8SClient.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
156+
157+
if domain, ok := node.Labels["node-domain"]; ok {
158+
nodeDomain = domain
159+
} else {
160+
slog.Warn("Node domain not found, using default domain")
161+
nodeDomain = store.GetSettingString("node_domain")
162+
}
163+
134164
ingress := &networkingv1.Ingress{
135165
ObjectMeta: metav1.ObjectMeta{
136-
Name: name + "-ingress",
137-
Labels: map[string]string{
138-
"challenge": c.UUID,
139-
"user": u.UUID,
140-
},
166+
Name: name + "-ingress",
167+
Labels: info.Labels,
141168
},
142169
Spec: networkingv1.IngressSpec{
143170
Rules: []networkingv1.IngressRule{
144171
{
145-
Host: fmt.Sprintf("%s.%s", containerUUID, store.GetSettingString("node_domain")),
172+
Host: fmt.Sprintf("%s.%s", containerUUID, nodeDomain),
146173
IngressRuleValue: networkingv1.IngressRuleValue{
147174
HTTP: &networkingv1.HTTPIngressRuleValue{
148175
Paths: []networkingv1.HTTPIngressPath{
@@ -153,7 +180,7 @@ func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *stor
153180
Service: &networkingv1.IngressServiceBackend{
154181
Name: name + "-service",
155182
Port: networkingv1.ServiceBackendPort{
156-
Number: int32(c.ExposedPort),
183+
Number: int32(payload.ExposedPort),
157184
},
158185
},
159186
},
@@ -169,15 +196,15 @@ func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *stor
169196
if err != nil {
170197
// handle error
171198
slog.Error(err.Error())
172-
return "", err
199+
return "", "", err
173200
}
174201

175202
slog.Info(fmt.Sprintf("Ingress created: %s", ingressObj.Name))
176-
return containerUUID, nil
203+
return containerUUID, nodeDomain, nil
177204
}
178205

179-
func (cm *ContainerManager) DisposeChallengeContainer(c *store.Challenge, u *store.User) error {
180-
hash := util.SHA256(c.UUID + u.UUID)[:16]
206+
func (cm *ContainerManager) DisposeContainer(identifier string) error {
207+
hash := util.SHA256(identifier)[:16]
181208
name := "container-" + hash
182209
// Delete a container in k8s
183210
err := cm.K8SClient.AppsV1().Deployments("challenge-containers").Delete(context.Background(), name, metav1.DeleteOptions{})
@@ -203,3 +230,29 @@ func (cm *ContainerManager) DisposeChallengeContainer(c *store.Challenge, u *sto
203230

204231
return nil
205232
}
233+
234+
func (cm *ContainerManager) CreateChallengeContainer(c *store.Challenge, u *store.User, flag string, s *store.Store) (string, string, error) {
235+
payload := &ContainerCreatePayload{
236+
Image: c.Image.Name,
237+
MemoryLimit: c.Image.MemoryLimit,
238+
CPULimit: c.Image.CPULimit,
239+
StorageLimit: c.Image.StorageLimit,
240+
ExposedPort: c.Image.ExposedPort,
241+
RegistryAccessTokenUUID: c.Image.RegistryAccessTokenUUID,
242+
}
243+
244+
info := &ContainerInfo{
245+
Identifier: c.UUID + u.UUID,
246+
Labels: map[string]string{
247+
"challenge": c.UUID,
248+
"user": u.UUID,
249+
},
250+
Flag: flag,
251+
}
252+
253+
return cm.CreateContainer(payload, info, s)
254+
}
255+
256+
func (cm *ContainerManager) DisposeChallengeContainer(c *store.Challenge, u *store.User) error {
257+
return cm.DisposeContainer(c.UUID + u.UUID)
258+
}

‎plugins/cron/container_cron.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func InitContainerCron(s *store.Store, cm *k8s.ContainerManager) {
3535
}
3636

3737
for _, container := range expiredContainers {
38-
if err := cm.DisposeChallengeContainer(container.Challenge, container.Creator); err != nil {
38+
if err := cm.DisposeContainer(container.Identifier); err != nil {
3939
slog.Error("Failed to delete container: " + err.Error())
4040
}
4141

‎server/router/api/v1/challenge_service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@ import (
2424
type CreateChallengePayload struct {
2525
Name string `json:"name" validate:"required"`
2626
Description string `json:"description" validate:"required"`
27-
Attachments []string `json:"attachments"`
2827
Category string `json:"category" validate:"required"`
2928
Tags []string `json:"tags"`
3029
ExpireTime int64 `json:"expire_time" validate:"required"`
3130
AfterExpiredOptions int32 `json:"after_expired_options" validate:"required"`
31+
Attachments []string `json:"attachments"`
3232

3333
Image string `json:"image"`
3434
MemoryLimit string `json:"memory_limit"`

‎server/router/api/v1/container_service.go‎

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,13 @@ import (
2424
"github.com/google/uuid"
2525
"github.com/labstack/echo/v4"
2626
"github.com/spf13/cast"
27+
"rina.icu/hoshino/internal/k8s"
28+
"rina.icu/hoshino/internal/util"
2729
"rina.icu/hoshino/server/context"
2830
"rina.icu/hoshino/store"
2931
)
3032

31-
func CreateContainer(c echo.Context) error {
33+
func CreateChallengeContainer(c echo.Context) error {
3234
ctx := c.(*context.CustomContext)
3335

3436
challenge, err := ctx.Store.GetChallengeByUUID(c.Param("challenge_uuid"))
@@ -62,14 +64,15 @@ func CreateContainer(c echo.Context) error {
6264

6365
slog.Info("Creating container for challenge %s", slog.Any("challenge", challenge))
6466

65-
uuid, err := ctx.ContainerManager.CreateChallengeContainer(challenge, user, flag, ctx.Store)
67+
uuid, nodeDomain, err := ctx.ContainerManager.CreateChallengeContainer(challenge, user, flag, ctx.Store)
6668
containerModel := &store.Container{
6769
Creator: user,
6870
Challenge: challenge,
6971
UUID: uuid,
7072
Status: store.ContainerStatusRunning,
7173
ExpireTime: time.Now().Unix() + cast.ToInt64(ctx.Store.GetSettingInt("container_expire_time")),
7274
LeftRenewalTimes: ctx.Store.GetSettingInt("max_container_renewal_times"),
75+
Identifier: challenge.UUID + user.UUID,
7376
}
7477

7578
flagModel := &store.Flag{
@@ -88,12 +91,12 @@ func CreateContainer(c echo.Context) error {
8891

8992
return OKWithData(&c, map[string]interface{}{
9093
"uuid": uuid,
91-
"entrance": fmt.Sprintf("%s.%s", uuid, ctx.Store.GetSettingString("node_domain")),
94+
"entrance": fmt.Sprintf("%s.%s", uuid, nodeDomain),
9295
"expire": containerModel.ExpireTime,
9396
})
9497
}
9598

96-
func DisposeContainer(c echo.Context) error {
99+
func DisposeChallengeContainer(c echo.Context) error {
97100
ctx := c.(*context.CustomContext)
98101

99102
challenge, err := ctx.Store.GetChallengeByUUID(c.Param("challenge_uuid"))
@@ -123,3 +126,82 @@ func DisposeContainer(c echo.Context) error {
123126

124127
return OK(&c)
125128
}
129+
130+
func CreateContainer(c echo.Context) error {
131+
ctx := c.(*context.CustomContext)
132+
133+
user, _ := GetUserFromToken(&c)
134+
game, _ := ctx.Store.GetGameByUUID(c.Param("game_uuid"))
135+
136+
if user.Privilege < store.UserPrivilegeAdministrator && !game.IsManager(user) {
137+
return PermissionDenied(&c)
138+
}
139+
140+
payload := new(k8s.ContainerCreatePayload)
141+
if err := c.Bind(payload); err != nil {
142+
return Failed(&c, "Bad request")
143+
}
144+
145+
identifier := "test-" + util.SHA256(util.UUID())[:16]
146+
147+
flag := fmt.Sprintf("%s{%s}", game.FlagPrefix, util.GenerateFlagContent(c.QueryParams().Get("flag_format"), user.UUID))
148+
// this is a test container, so we don't need to create the flag model
149+
150+
info := &k8s.ContainerInfo{
151+
Identifier: identifier,
152+
Labels: map[string]string{
153+
"test": "true",
154+
"game": game.UUID,
155+
"user": user.UUID,
156+
},
157+
Flag: flag,
158+
}
159+
160+
uuid, nodeDomain, err := ctx.ContainerManager.CreateContainer(payload, info, ctx.Store)
161+
162+
if err != nil {
163+
return Failed(&c, "Unable to create container.")
164+
}
165+
166+
container := &store.Container{
167+
Creator: user,
168+
UUID: uuid,
169+
Status: store.ContainerStatusRunning,
170+
ExpireTime: time.Now().Unix() + cast.ToInt64(ctx.Store.GetSettingInt("container_expire_time")),
171+
LeftRenewalTimes: ctx.Store.GetSettingInt("max_container_renewal_times"),
172+
Identifier: identifier,
173+
}
174+
175+
ctx.Store.CreateContainer(container)
176+
177+
return OKWithData(&c, map[string]interface{}{
178+
"identifier": identifier,
179+
"uuid": uuid,
180+
"entrance": fmt.Sprintf("%s.%s", uuid, nodeDomain),
181+
"flag": flag,
182+
})
183+
}
184+
185+
func DisposeContainer(c echo.Context) error {
186+
ctx := c.(*context.CustomContext)
187+
188+
user, _ := GetUserFromToken(&c)
189+
game, _ := ctx.Store.GetGameByUUID(c.Param("game_uuid"))
190+
191+
if user.Privilege < store.UserPrivilegeAdministrator && !game.IsManager(user) {
192+
return PermissionDenied(&c)
193+
}
194+
195+
container, err := ctx.Store.GetContainerByUUID(c.Param("container_uuid"))
196+
197+
if err != nil {
198+
return Failed(&c, "Unable to fetch container")
199+
}
200+
201+
ctx.ContainerManager.DisposeContainer(container.Identifier)
202+
203+
container.Status = store.ContainerStatusStopped
204+
ctx.Store.UpdateContainer(container)
205+
206+
return OK(&c)
207+
}

0 commit comments

Comments
 (0)