OADP-5901: DataProtectionTest CRD and Controller Implementation (#1714)

* Add DPT CRD and controller

* Add s3 vendor determination

* Add s3 vendor determination tests

* Add cloud provider interface

* Implement cloud provider interface for AWS

* Add untils

* Initialize provider and perform upload speed test

* Handle aws profile from config

* Resolve the backup location spec to be used from DPT

* Add functionality to fetch bucket metadata: versioning and encryption

* Add forceRun spec support, update spec timeouts datatype to metav1.Duration, fix aws vendor bug

* update vendor determination and tests, minor fixes

* Add unit tests for resolveBackupLocation, initializeProvider, runUploadTest and GetBucketMetadata

* Add runSnapshotTests functionality

* Code hardening and logging

* Error handling for runSnapshotTests

* Reconcile loop Error handling at DPT status level

* Add printerColumns to DPT

* Handle DPT status phase marking

* Add snapshot.storage.k8s.io perms

* Add snapshot summary

* minor fixes

* lint fix

Author: shubham-pampattiwar

PROJECT

@@ -29,4 +29,13 @@ resources:
   kind: CloudStorage
   path: github.com/openshift/oadp-operator/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: openshift.io
+  group: oadp
+  kind: DataProtectionTest
+  path: github.com/openshift/oadp-operator/api/v1alpha1
+  version: v1alpha1
 version: "3"

api/v1alpha1/dataprotectiontest_types.go

@@ -0,0 +1,211 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// DataProtectionTestSpec defines the desired tests to perform.
+type DataProtectionTestSpec struct {
+	// backupLocationName specifies the name the Velero BackupStorageLocation (BSL) to test against.
+	// +optional
+	BackupLocationName string `json:"backupLocationName,omitempty"`
+
+	// backupLocationSpec is an inline copy of the BSL spec to use during testing.
+	// +optional
+	BackupLocationSpec *velerov1.BackupStorageLocationSpec `json:"backupLocationSpec,omitempty"`
+
+	// uploadSpeedTestConfig specifies parameters for an object storage upload speed test.
+	// +optional
+	UploadSpeedTestConfig *UploadSpeedTestConfig `json:"uploadSpeedTestConfig,omitempty"`
+
+	// csiVolumeSnapshotTestConfigs defines one or more CSI VolumeSnapshot tests to perform.
+	// +optional
+	CSIVolumeSnapshotTestConfigs []CSIVolumeSnapshotTestConfig `json:"csiVolumeSnapshotTestConfigs,omitempty"`
+
+	// forceRun will re-trigger the DPT even if it already completed
+	// +kubebuilder:default=false
+	// +optional
+	ForceRun bool `json:"forceRun,omitempty"`
+}
+
+// UploadSpeedTestConfig contains configuration for testing object storage upload performance.
+type UploadSpeedTestConfig struct {
+	// fileSize is the size of data to upload, e.g., "100MB".
+	// +optional
+	FileSize string `json:"fileSize,omitempty"`
+
+	// timeout defines the maximum duration for the upload test, e.g., "60s".
+	// +optional
+	Timeout metav1.Duration `json:"timeout,omitempty"`
+}
+
+// CSIVolumeSnapshotTestConfig contains config for performing a CSI VolumeSnapshot test.
+type CSIVolumeSnapshotTestConfig struct {
+	// snapshotClassName specifies the CSI snapshot class to use.
+	// +optional
+	SnapshotClassName string `json:"snapshotClassName,omitempty"`
+
+	// timeout specifies how long to wait for the snapshot to become ready, e.g., "60s"
+	// +optional
+	Timeout metav1.Duration `json:"timeout,omitempty"`
+
+	// volumeSnapshotSource defines the PVC to snapshot.
+	// +optional
+	VolumeSnapshotSource VolumeSnapshotSource `json:"volumeSnapshotSource,omitempty"`
+}
+
+// VolumeSnapshotSource points to the PVC that should be snapshotted.
+type VolumeSnapshotSource struct {
+	// persistentVolumeClaimName is the name of the PVC to snapshot.
+	// +optional
+	PersistentVolumeClaimName string `json:"persistentVolumeClaimName,omitempty"`
+
+	// persistentVolumeClaimNamespace is the namespace of the PVC.
+	// +optional
+	PersistentVolumeClaimNamespace string `json:"persistentVolumeClaimNamespace,omitempty"`
+}
+
+// DataProtectionTestStatus represents the observed results of the tests.
+type DataProtectionTestStatus struct {
+	// lastTested is the timestamp when the test was last run.
+	// +optional
+	LastTested metav1.Time `json:"lastTested,omitempty"`
+
+	// s3Vendor indicates the detected s3 vendor name from the storage endpoint if applicable (e.g., AWS, MinIO).
+	// +optional
+	S3Vendor string `json:"s3Vendor,omitempty"`
+
+	// bucketMetadata reports the encryption and versioning status of the target bucket.
+	// +optional
+	BucketMetadata *BucketMetadata `json:"bucketMetadata,omitempty"`
+
+	// uploadTest contains results of the object storage upload test.
+	// +optional
+	UploadTest UploadTestStatus `json:"uploadTest,omitempty"`
+
+	// snapshotTests contains results for each snapshot tested PVC.
+	// +optional
+	SnapshotTests []SnapshotTestStatus `json:"snapshotTests,omitempty"`
+
+	// snapshot test pass/fail summary
+	// +optional
+	SnapshotSummary string `json:"snapshotSummary,omitempty"`
+
+	// phase indicates phase of the DataProtectionTest - Complete, Failed
+	// +optional
+	Phase string `json:"phase,omitempty"`
+
+	// errorMessage contains details of any DPT failure
+	// +optional
+	ErrorMessage string `json:"errorMessage,omitempty"`
+}
+
+// UploadTestStatus holds the results of the upload test.
+type UploadTestStatus struct {
+	// speedMbps is the calculated upload speed.
+	// +optional
+	SpeedMbps int64 `json:"speedMbps,omitempty"`
+
+	// duration is the time taken to upload the test file.
+	// +optional
+	Duration string `json:"duration,omitempty"`
+
+	// success indicates if the upload succeeded.
+	// +optional
+	Success bool `json:"success,omitempty"`
+
+	// errorMessage contains details of any upload failure.
+	// +optional
+	ErrorMessage string `json:"errorMessage,omitempty"`
+}
+
+// SnapshotTestStatus holds the result for an individual PVC snapshot test.
+type SnapshotTestStatus struct {
+	// persistentVolumeClaimName of the tested PVC.
+	// +optional
+	PersistentVolumeClaimName string `json:"persistentVolumeClaimName,omitempty"`
+
+	// persistentVolumeClaimNamespace of the tested PVC.
+	// +optional
+	PersistentVolumeClaimNamespace string `json:"persistentVolumeClaimNamespace,omitempty"`
+
+	// status indicates snapshot readiness ("Ready", "Failed").
+	// +optional
+	Status string `json:"status,omitempty"`
+
+	// readyDuration is the time it took for the snapshot to become ReadyToUse.
+	// +optional
+	ReadyDuration string `json:"readyDuration,omitempty"`
+
+	// errorMessage contains details of any snapshot failure.
+	// +optional
+	ErrorMessage string `json:"errorMessage,omitempty"`
+}
+
+// BucketMetadata contains encryption and versioning info for the target bucket.
+type BucketMetadata struct {
+	// encryptionAlgorithm reports the encryption method (AES256, aws:kms, or "None").
+	// +optional
+	EncryptionAlgorithm string `json:"encryptionAlgorithm,omitempty"`
+
+	// versioningStatus indicates whether bucket versioning is Enabled, Suspended, or None.
+	// +optional
+	VersioningStatus string `json:"versioningStatus,omitempty"`
+
+	// errorMessage contains details of any failure to fetch bucket metadata.
+	// +optional
+	ErrorMessage string `json:"errorMessage,omitempty"`
+}
+
+// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=".status.phase",description="Current phase of the DPT"
+// +kubebuilder:printcolumn:name="LastTested",type=date,JSONPath=".status.lastTested",description="Last time the test was executed"
+// +kubebuilder:printcolumn:name="UploadSpeed(Mbps)",type=integer,JSONPath=".status.uploadTest.speedMbps",description="Upload speed to object storage"
+// +kubebuilder:printcolumn:name="Encryption",type=string,JSONPath=".status.bucketMetadata.encryptionAlgorithm",description="Bucket encryption algorithm"
+// +kubebuilder:printcolumn:name="Versioning",type=string,JSONPath=".status.bucketMetadata.versioningStatus",description="Bucket versioning state"
+// +kubebuilder:printcolumn:name="Snapshots",type=string,JSONPath=`.status.snapshotSummary`,description="Snapshot test pass/fail summary"
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp",description="Time since DPT was created"
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:resource:path=dataprotectiontests,shortName=dpt
+
+// DataProtectionTest is the Schema for the dataprotectiontests API
+type DataProtectionTest struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DataProtectionTestSpec   `json:"spec,omitempty"`
+	Status DataProtectionTestStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// DataProtectionTestList contains a list of DataProtectionTest
+type DataProtectionTestList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []DataProtectionTest `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&DataProtectionTest{}, &DataProtectionTestList{})
+}