e2e improvements

[OKE-36695][OKE-36696] e2e logging improvements

[OKE-37816] Logging improvement for RWX tests

[OKE-37639] Increase slow pod start timeout in e2es

[OKE-37370] Update boot volume test to clone boot volume in compartment

Author: YashasG98

test/e2e/cloud-provider-oci/boot_volume.go

@@ -46,7 +46,7 @@ var _ = Describe("Boot volume tests", func() {
 			opts := framework.Options{
 				BlockProvisionerName: setupF.BlockProvisionerName,
 			}
-			pvc, bootvolumeId := pvcJig.CreateAndAwaitStaticBootVolumePVCOrFailCSI(f.ComputeClient, f.Namespace.Name, compartmentId, setupF.AdLocation, setupF.MntTargetSubnetOcid, framework.MinVolumeBlock, scName, nil, v1.PersistentVolumeBlock, v1.ReadWriteOnce, v1.ClaimPending, opts)
+			pvc, bootvolumeId := pvcJig.CreateAndAwaitStaticBootVolumePVCOrFailCSI(f.ComputeClient, f.BlockStorageClient, f.Namespace.Name, compartmentId, setupF.AdLocation, framework.MinVolumeBlock, scName, nil, v1.PersistentVolumeBlock, v1.ReadWriteOnce, v1.ClaimPending, opts)
 			f.VolumeIds = append(f.VolumeIds, pvc.Spec.VolumeName)
 
 			podName := pvcJig.NewPodForCSI("app1", f.Namespace.Name, pvc.Name, setupF.AdLabel, v1.PersistentVolumeBlock)
@@ -74,13 +74,10 @@ var _ = Describe("Boot volume gating test", func() {
 				framework.Failf("Compartment Id undefined.")
 			}
 
-			scName := f.CreateStorageClassOrFail(f.Namespace.Name, setupF.BlockProvisionerName,
+			scName := f.CreateStorageClassOrFail(f.Namespace.Name, "blockvolume.csi.oraclecloud.com",
 				map[string]string{framework.AttachmentType: framework.AttachmentTypeISCSI},
 				pvcJig.Labels, "WaitForFirstConsumer", false, "Retain", nil)
-			opts := framework.Options{
-				BlockProvisionerName: setupF.BlockProvisionerName,
-			}
-			pvc, bootvolumeId := pvcJig.CreateAndAwaitStaticBootVolumePVCOrFailCSI(f.ComputeClient, f.Namespace.Name, compartmentId, setupF.AdLocation, setupF.MntTargetSubnetOcid, framework.MinVolumeBlock, scName, nil, v1.PersistentVolumeFilesystem, v1.ReadWriteOnce, v1.ClaimPending, opts)
+			pvc, bootvolumeId := pvcJig.CreateAndAwaitStaticBootVolumePVCOrFailCSI(f.ComputeClient, f.BlockStorageClient, f.Namespace.Name, compartmentId, setupF.AdLocation, framework.MinVolumeBlock, scName, nil, v1.PersistentVolumeFilesystem, v1.ReadWriteOnce, v1.ClaimPending)
 			pvcJig.NewPodForCSIWithoutWait("app1", f.Namespace.Name, pvc.Name, setupF.AdLabel)
 			err := pvcJig.WaitTimeoutForPodRunningInNamespace("app1", f.Namespace.Name, 7*time.Minute)
 			if err == nil {

test/e2e/framework/cloud_provider_framework.go

@@ -343,6 +343,21 @@ func (f *CloudProviderFramework) AfterEach() {
 	if len(nsDeletionErrors) != 0 {
 		messages := []string{}
 		for namespaceKey, namespaceErr := range nsDeletionErrors {
+			if strings.Contains(namespaceErr.Error(), "timed out waiting for the condition") {
+				Logf("Couldn't delete ns: %q: %s", namespaceKey, namespaceErr.Error())
+				pods, err := f.ClientSet.CoreV1().Pods(namespaceKey).List(context.TODO(), metav1.ListOptions{})
+				if err != nil {
+					Logf("Failed to list pods in ns %q: %v", namespaceKey, err)
+				} else {
+					for _, pod := range pods.Items {
+						if pod.DeletionTimestamp != nil {
+							Logf("Pod %q is stuck terminating in namespace %q", pod.Name, namespaceKey)
+							pvcJig := NewPVCTestJig(f.ClientSet, "namespace-stuck-pod-logging")
+							pvcJig.LogPodDebugInfo(namespaceKey, pod.Name)
+						}
+					}
+				}
+			}
 			messages = append(messages, fmt.Sprintf("Couldn't delete ns: %q: %s (%#v)", namespaceKey, namespaceErr, namespaceErr))
 		}
 		Failf(strings.Join(messages, ","))

test/e2e/framework/pod_util.go

@@ -408,6 +408,8 @@ func (j *PVCTestJig) CreateAndAwaitNginxPodOrFail(ns string, pvc *v1.PersistentV
 	// Waiting for pod to be running
 	err = j.WaitTimeoutForPodRunningInNamespace(pod.Name, ns, slowPodStartTimeout)
 	if err != nil {
+		Logf("Pod failed to come up, logging debug info\n")
+		j.LogPodDebugInfo(namespace, pod.Name)
 		Failf("Pod %q is not Running: %v", pod.Name, err)
 	}
 
@@ -704,10 +706,17 @@ func (j *PVCTestJig) logNodeDriverLogs(pod *v1.Pod) {
 	}
 }
 
-func (j *PVCTestJig) logPodDebugInfo(ns string, podName string) {
+func (j *PVCTestJig) LogPodDebugInfo(ns string, podName string) {
 	pod, err := j.describePod(ns, podName)
 	if err != nil {
 		Logf("Error describing pod: %v", err)
+		if apierrors.IsNotFound(err) {
+			pods, listErr := j.KubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
+			if listErr != nil {
+				Logf("Error listing pods: %v", listErr)
+			}
+			Logf("Pods in namespace list: %v", pods.Items)
+		}
 		return
 	}