build/ci: handle split up listing (#21293)

Author: holiman

build/ci.go

@@ -1098,6 +1098,8 @@ func doPurge(cmdline []string) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	fmt.Printf("Found %d blobs\n", len(blobs))
+
 	// Iterate over the blobs, collect and sort all unstable builds
 	for i := 0; i < len(blobs); i++ {
 		if !strings.Contains(blobs[i].Name, "unstable") {
@@ -1119,6 +1121,7 @@ func doPurge(cmdline []string) {
 			break
 		}
 	}
+	fmt.Printf("Deleting %d blobs\n", len(blobs))
 	// Delete all marked as such and return
 	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
 		log.Fatal(err)

internal/build/azure.go

@@ -71,26 +71,35 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
 
 // AzureBlobstoreList lists all the files contained within an azure blobstore.
 func AzureBlobstoreList(config AzureBlobstoreConfig) ([]azblob.BlobItem, error) {
-	credential, err := azblob.NewSharedKeyCredential(config.Account, config.Token)
-	if err != nil {
-		return nil, err
+	credential := azblob.NewAnonymousCredential()
+	if len(config.Token) > 0 {
+		c, err := azblob.NewSharedKeyCredential(config.Account, config.Token)
+		if err != nil {
+			return nil, err
+		}
+		credential = c
 	}
-
 	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
 
 	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account))
 	service := azblob.NewServiceURL(*u, pipeline)
 
+	var allBlobs []azblob.BlobItem
 	// List all the blobs from the container and return them
 	container := service.NewContainerURL(config.Container)
+	nextMarker := azblob.Marker{}
+	for nextMarker.NotDone() {
+		res, err := container.ListBlobsFlatSegment(context.Background(), nextMarker, azblob.ListBlobsSegmentOptions{
+			MaxResults: 5000, // The server only gives max 5K items
+		})
+		if err != nil {
+			return nil, err
+		}
+		allBlobs = append(allBlobs, res.Segment.BlobItems...)
+		nextMarker = res.NextMarker
 
-	res, err := container.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{
-		MaxResults: 1024 * 1024 * 1024, // Yes, fetch all of them
-	})
-	if err != nil {
-		return nil, err
 	}
-	return res.Segment.BlobItems, nil
+	return allBlobs, nil
 }
 
 // AzureBlobstoreDelete iterates over a list of files to delete and removes them
@@ -121,6 +130,7 @@ func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem)
 		if _, err := blockblob.Delete(context.Background(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
 			return err
 		}
+		fmt.Printf("deleted  %s (%s)\n", blob.Name, blob.Properties.LastModified)
 	}
 	return nil
 }