Skip to content

Commit 27b69c7

Browse files
amstutaamstuta
committed
feat: Add options to s3 objects bulk-delete command
Signed-off-by: Arthur Amstutz <arthur.amstutz@corp.ovh.com>
1 parent 5cbfefd commit 27b69c7

File tree

4 files changed

+249
-22
lines changed

4 files changed

+249
-22
lines changed

‎doc/ovhcloud_cloud_storage-s3_bulk-delete.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ ovhcloud cloud storage-s3 bulk-delete <container_name> [flags]
99
### Options
1010

1111
```
12+
--all Delete all objects in the container
1213
-h, --help help for bulk-delete
1314
--objects strings List of objects to delete (format is '<object_name>' or '<object_name>:<version_id>'
15+
--prefix string Prefix to filter objects to delete
1416
```
1517

1618
### Options inherited from parent commands

‎internal/cmd/cloud_storage_s3.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ func initCloudStorageS3Command(cloudCmd *cobra.Command) {
6666
Args: cobra.ExactArgs(1),
6767
}
6868
bulkDeleteCmd.Flags().StringSliceVar(&cloud.StorageS3ObjectsToDelete, "objects", nil, "List of objects to delete (format is '<object_name>' or '<object_name>:<version_id>'")
69+
bulkDeleteCmd.Flags().BoolVar(&cloud.StorageS3BulkDeleteAll, "all", false, "Delete all objects in the container")
70+
bulkDeleteCmd.Flags().StringVar(&cloud.StorageS3BulkDeletePrefix, "prefix", "", "Prefix to filter objects to delete")
71+
bulkDeleteCmd.MarkFlagsOneRequired("objects", "all", "prefix")
72+
bulkDeleteCmd.MarkFlagsMutuallyExclusive("objects", "all", "prefix")
73+
6974
storageS3Cmd.AddCommand(bulkDeleteCmd)
7075

7176
// Object commands

‎internal/cmd/cloud_storage_s3_test.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// SPDX-FileCopyrightText: 2025 OVH SAS <opensource@ovh.net>
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
package cmd_test
6+
7+
import (
8+
"encoding/json"
9+
"net/http"
10+
11+
"github.com/jarcoal/httpmock"
12+
"github.com/maxatome/go-testdeep/td"
13+
"github.com/maxatome/tdhttpmock"
14+
"github.com/ovh/ovhcloud-cli/internal/cmd"
15+
)
16+
17+
func (ms *MockSuite) TestCloudStorageS3BulkDeletePrefixCmd(assert, require *td.T) {
18+
httpmock.RegisterResponder(http.MethodGet, "https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region",
19+
httpmock.NewStringResponder(200, `["BHS"]`))
20+
21+
httpmock.RegisterResponder(http.MethodGet, "https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS",
22+
httpmock.NewStringResponder(200, `{
23+
"name": "BHS",
24+
"type": "region",
25+
"status": "UP",
26+
"services": [
27+
{
28+
"name": "storage",
29+
"status": "UP"
30+
},
31+
{
32+
"name": "storage-s3-high-perf",
33+
"status": "UP"
34+
},
35+
{
36+
"name": "storage-s3-standard",
37+
"status": "UP"
38+
}
39+
],
40+
"countryCode": "ca",
41+
"ipCountries": [],
42+
"continentCode": "NA",
43+
"availabilityZones": [],
44+
"datacenterLocation": "BHS"
45+
}`))
46+
47+
httpmock.RegisterResponder(http.MethodGet,
48+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer",
49+
httpmock.NewStringResponder(200, `{
50+
"name": "fakeContainer",
51+
"virtualHost": "https://fakeContainer.test.ovh.net/",
52+
"ownerId": 0,
53+
"objectsCount": 15,
54+
"objectsSize": 4147089,
55+
"objects": [
56+
{"key": "logs/log1.txt"},
57+
{"key": "logs/log2.txt"},
58+
{"key": "images/img1.png"}
59+
],
60+
"region": "BHS",
61+
"createdAt": "2025-02-10T14:24:12Z"
62+
}`))
63+
64+
httpmock.RegisterResponder(http.MethodGet,
65+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer/object?prefix=logs%2F",
66+
httpmock.NewStringResponder(200, `[
67+
{"key": "logs/log1.txt"},
68+
{"key": "logs/log2.txt"}
69+
]`).Then(httpmock.NewStringResponder(200, `[]`)),
70+
)
71+
72+
httpmock.RegisterMatcherResponder(http.MethodPost,
73+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer/bulkDeleteObjects",
74+
tdhttpmock.JSONBody(td.JSON(`
75+
{
76+
"objects": [
77+
{"key": "logs/log1.txt"},
78+
{"key": "logs/log2.txt"}
79+
]
80+
}`),
81+
),
82+
httpmock.NewStringResponder(200, ``),
83+
)
84+
85+
out, err := cmd.Execute("cloud", "storage-s3", "bulk-delete", "fakeContainer", "--cloud-project", "fakeProjectID", "--prefix", "logs/", "--json")
86+
require.CmpNoError(err)
87+
assert.Cmp(json.RawMessage(out), td.JSON(`{"message": "✅ Objects deleted successfully"}`))
88+
}
89+
90+
func (ms *MockSuite) TestCloudStorageS3BulkDeleteAllCmd(assert, require *td.T) {
91+
httpmock.RegisterResponder(http.MethodGet, "https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region",
92+
httpmock.NewStringResponder(200, `["BHS"]`))
93+
94+
httpmock.RegisterResponder(http.MethodGet, "https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS",
95+
httpmock.NewStringResponder(200, `{
96+
"name": "BHS",
97+
"type": "region",
98+
"status": "UP",
99+
"services": [
100+
{
101+
"name": "storage",
102+
"status": "UP"
103+
},
104+
{
105+
"name": "storage-s3-high-perf",
106+
"status": "UP"
107+
},
108+
{
109+
"name": "storage-s3-standard",
110+
"status": "UP"
111+
}
112+
],
113+
"countryCode": "ca",
114+
"ipCountries": [],
115+
"continentCode": "NA",
116+
"availabilityZones": [],
117+
"datacenterLocation": "BHS"
118+
}`))
119+
120+
httpmock.RegisterResponder(http.MethodGet,
121+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer",
122+
httpmock.NewStringResponder(200, `{
123+
"name": "fakeContainer",
124+
"virtualHost": "https://fakeContainer.test.ovh.net/",
125+
"ownerId": 0,
126+
"objectsCount": 15,
127+
"objectsSize": 4147089,
128+
"objects": [
129+
{"key": "logs/log1.txt"},
130+
{"key": "logs/log2.txt"},
131+
{"key": "images/img1.png"}
132+
],
133+
"region": "BHS",
134+
"createdAt": "2025-02-10T14:24:12Z"
135+
}`))
136+
137+
httpmock.RegisterResponder(http.MethodGet,
138+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer/object",
139+
httpmock.NewStringResponder(200, `[
140+
{"key": "logs/log1.txt"},
141+
{"key": "logs/log2.txt"},
142+
{"key": "images/img1.png"}
143+
]`).Then(httpmock.NewStringResponder(200, `[]`)),
144+
)
145+
146+
httpmock.RegisterMatcherResponder(http.MethodPost,
147+
"https://eu.api.ovh.com/1.0/cloud/project/fakeProjectID/region/BHS/storage/fakeContainer/bulkDeleteObjects",
148+
tdhttpmock.JSONBody(td.JSON(`
149+
{
150+
"objects": [
151+
{"key": "logs/log1.txt"},
152+
{"key": "logs/log2.txt"},
153+
{"key": "images/img1.png"}
154+
]
155+
}`),
156+
),
157+
httpmock.NewStringResponder(200, ``),
158+
)
159+
160+
out, err := cmd.Execute("cloud", "storage-s3", "bulk-delete", "fakeContainer", "--cloud-project", "fakeProjectID", "--all", "--json")
161+
require.CmpNoError(err)
162+
assert.Cmp(json.RawMessage(out), td.JSON(`{"message": "✅ Objects deleted successfully"}`))
163+
}

‎internal/services/cloud/cloud_storage_s3.go‎

Lines changed: 79 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
_ "embed"
99
"encoding/json"
1010
"fmt"
11+
"log"
12+
"net/http"
1113
"net/url"
1214
"strconv"
1315
"strings"
@@ -72,7 +74,9 @@ var (
7274
} `json:"versioning,omitzero"`
7375
}
7476

75-
StorageS3ObjectsToDelete []string
77+
StorageS3ObjectsToDelete []string
78+
StorageS3BulkDeleteAll bool
79+
StorageS3BulkDeletePrefix string
7680

7781
StorageS3ListParams struct {
7882
KeyMarker string
@@ -264,41 +268,94 @@ func StorageS3BulkDeleteObjects(_ *cobra.Command, args []string) {
264268
return
265269
}
266270

267-
if len(StorageS3ObjectsToDelete) == 0 {
268-
display.OutputWarning(&flags.OutputFormatConfig, "no objects to delete. Use --objects flag to specify objects to delete")
269-
return
270-
}
271-
272271
foundURL, _, err := locateStorageS3Container(projectID, args[0])
273272
if err != nil {
274273
display.OutputError(&flags.OutputFormatConfig, "%s", err)
275274
return
276275
}
277276

278-
var objectsToDelete []map[string]any
279-
for _, object := range StorageS3ObjectsToDelete {
280-
parts := strings.Split(object, ":")
277+
// List of objects to delete given, process them
278+
if len(StorageS3ObjectsToDelete) > 0 {
279+
var objectsToDelete []map[string]any
280+
for _, object := range StorageS3ObjectsToDelete {
281+
parts := strings.Split(object, ":")
281282

282-
switch len(parts) {
283-
case 1:
284-
// Object name only
285-
objectsToDelete = append(objectsToDelete, map[string]any{"key": parts[0]})
286-
case 2:
287-
// Object name with version ID
288-
objectsToDelete = append(objectsToDelete, map[string]any{"key": parts[0], "versionId": parts[1]})
289-
default:
290-
display.OutputError(&flags.OutputFormatConfig, "invalid object format: %s. Use <object_name> or <object_name>:<version_id>", object)
283+
switch len(parts) {
284+
case 1:
285+
// Object name only
286+
objectsToDelete = append(objectsToDelete, map[string]any{"key": parts[0]})
287+
case 2:
288+
// Object name with version ID
289+
objectsToDelete = append(objectsToDelete, map[string]any{"key": parts[0], "versionId": parts[1]})
290+
default:
291+
display.OutputError(&flags.OutputFormatConfig, "invalid object format: %s. Use <object_name> or <object_name>:<version_id>", object)
292+
return
293+
}
294+
}
295+
296+
if err := httpLib.Client.Post(foundURL+"/bulkDeleteObjects", map[string]any{
297+
"objects": objectsToDelete,
298+
}, nil); err != nil {
299+
display.OutputError(&flags.OutputFormatConfig, "failed to delete objects: %s", err)
291300
return
292301
}
302+
303+
display.OutputInfo(&flags.OutputFormatConfig, nil, "✅ Objects deleted successfully")
304+
return
293305
}
294306

295-
if err := httpLib.Client.Post(foundURL+"/bulkDeleteObjects", map[string]any{
296-
"objects": objectsToDelete,
297-
}, nil); err != nil {
298-
display.OutputError(&flags.OutputFormatConfig, "failed to delete objects: %s", err)
307+
var request *http.Request
308+
switch {
309+
case StorageS3BulkDeletePrefix != "":
310+
endpoint := foundURL + "/object?prefix=" + url.QueryEscape(StorageS3BulkDeletePrefix)
311+
request, err = httpLib.Client.NewRequest(http.MethodGet, endpoint, nil, true)
312+
case StorageS3BulkDeleteAll:
313+
request, err = httpLib.Client.NewRequest(http.MethodGet, foundURL+"/object", nil, true)
314+
default:
315+
display.OutputError(&flags.OutputFormatConfig, "Nothing to delete, either --objects, --prefix or --all must be specified")
316+
return
317+
}
318+
319+
if err != nil {
320+
display.OutputError(&flags.OutputFormatConfig, "failed to create objects listing request: %s", err)
299321
return
300322
}
301323

324+
for {
325+
// Fetch objects in the container (batches of 1000)
326+
resp, err := httpLib.Client.Do(request)
327+
if err != nil {
328+
display.OutputError(&flags.OutputFormatConfig, "failed to fetch objects: %s", err)
329+
return
330+
}
331+
332+
var objects []map[string]any
333+
if err := httpLib.Client.UnmarshalResponse(resp, &objects); err != nil {
334+
display.OutputError(&flags.OutputFormatConfig, "failed to parse objects response: %s", err)
335+
return
336+
}
337+
338+
// No objects found, we are done
339+
if len(objects) == 0 {
340+
break
341+
}
342+
343+
// Prepare objects to delete
344+
var objectsToDelete []map[string]any
345+
for _, object := range objects {
346+
objectsToDelete = append(objectsToDelete, map[string]any{"key": object["key"]})
347+
}
348+
349+
// Delete objects
350+
log.Printf("Deleting %d objects...", len(objectsToDelete))
351+
if err := httpLib.Client.Post(foundURL+"/bulkDeleteObjects", map[string]any{
352+
"objects": objectsToDelete,
353+
}, nil); err != nil {
354+
display.OutputError(&flags.OutputFormatConfig, "failed to delete objects: %s", err)
355+
return
356+
}
357+
}
358+
302359
display.OutputInfo(&flags.OutputFormatConfig, nil, "✅ Objects deleted successfully")
303360
}
304361

0 commit comments

Comments
 (0)